Merge branch 'RED-7384' into 'main'

Red 7384 See merge request fforesight/layout-parser!140
Red 7384
2024-04-23 12:13:21 +02:00 · 2024-04-23 12:13:19 +02:00 · 2024-04-19 09:21:50 +02:00 · 2024-04-18 17:42:34 +02:00 · 2024-04-18 15:08:00 +02:00 · 2024-04-18 14:39:10 +02:00
63 changed files with 1091 additions and 1273 deletions
@@ -42,3 +42,5 @@ gradlew.bat
 gradlew
 gradle.properties
 gradle/
+.DS_Store
+.DS_Store/
@@ -3,6 +3,5 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 public class LayoutParsingQueueNames {

    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
-    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_dead_letter_queue";
    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
 }
@@ -2,9 +2,9 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;

 public enum LayoutParsingType {
    REDACT_MANAGER,
-    TAAS,
+    REDACT_MANAGER_OLD,
+    REDACT_MANAGER_PARAGRAPH_DEBUG,
    DOCUMINE,
-
-    DOCSTRUM,
-    DOCSTRUM_XY
+    CLARIFYND,
+    CLARIFYND_PARAGRAPH_DEBUG
 }
@@ -24,4 +24,6 @@ dependencies {
    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
+    implementation("org.jgrapht:jgrapht-core:1.5.2")
+    implementation("org.tinspin:tinspin-indexes:2.1.3")
 }
@@ -47,10 +47,9 @@ import com.knecon.fforesight.service.layoutparser.processor.services.TableExtrac
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClarifyndClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
@@ -78,20 +77,19 @@ public class LayoutParsingPipeline {
    CvTableParsingAdapter cvTableParsingAdapter;
    LayoutParsingStorageService layoutParsingStorageService;
    SectionsBuilderService sectionsBuilderService;
-    TaasClassificationService taasClassificationService;
    RedactManagerClassificationService redactManagerClassificationService;
    DocuMineClassificationService docuMineClassificationService;
    SimplifiedSectionTextService simplifiedSectionTextService;
    BodyTextFrameService bodyTextFrameService;
    RulingCleaningService rulingCleaningService;
    TableExtractionService tableExtractionService;
-    TaasBlockificationService taasBlockificationService;
    DocuMineBlockificationService docuMineBlockificationService;
    RedactManagerBlockificationService redactManagerBlockificationService;
    DocstrumBlockificationService docstrumBlockificationService;
    LayoutGridService layoutGridService;
    ObservationRegistry observationRegistry;
    VisualLayoutParsingAdapter visualLayoutParsingAdapter;
+    ClarifyndClassificationService clarifyndClassificationService;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -100,40 +98,33 @@ public class LayoutParsingPipeline {
        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId())
-                .orElse(originFile);
+        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);

        VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse();
-        if (layoutParsingRequest.visualLayoutParsingFileId()
-                .isPresent()) {
-            visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId()
-                                                                                                         .get());
+        if (layoutParsingRequest.visualLayoutParsingFileId().isPresent()) {
+            visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId().get());
        }

        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId()
-                .isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId()
-                                                                                     .get());
+        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
        }

        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId()
-                .isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId()
-                                                                                     .get());
+        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
        }

        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
-                                                                    originFile,
-                                                                    imageServiceResponse,
-                                                                    tableServiceResponse,
-                                                                    visualLayoutParsingResponse,
-                layoutParsingRequest.identifier().toString());
+                originFile,
+                imageServiceResponse,
+                tableServiceResponse,
+                visualLayoutParsingResponse,
+                layoutParsingRequest.identifier());

        log.info("Building document graph for {}", layoutParsingRequest.identifier());

-        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
+        Document documentGraph = observeBuildDocumentGraph(layoutParsingRequest.layoutParsingType(), classificationDocument);

        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());

@@ -145,7 +136,7 @@ public class LayoutParsingPipeline {
        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);

-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.CLARIFYND)) {
            log.info("Building research document data for {}", layoutParsingRequest.identifier());
            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
@@ -161,37 +152,37 @@ public class LayoutParsingPipeline {
                .numberOfPages(documentGraph.getNumberOfPages())
                .duration(System.currentTimeMillis() - start)
                .message(format("""
-                                        Layout parsing has finished in %.02f s.
-                                        identifiers: %s
-                                        %s
-                                        Files have been saved with Ids:
-                                        Structure: %s
-                                        Text: %s
-                                        Positions: %s
-                                        PageData: %s
-                                        Simplified Text: %s
-                                        Viewer Doc: %s""",
-                                ((float) (System.currentTimeMillis() - start)) / 1000,
-                                layoutParsingRequest.identifier(),
-                                buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
-                                layoutParsingRequest.structureFileStorageId(),
-                                layoutParsingRequest.textBlockFileStorageId(),
-                                layoutParsingRequest.positionBlockFileStorageId(),
-                                layoutParsingRequest.pageFileStorageId(),
-                                layoutParsingRequest.simplifiedTextStorageId(),
-                                layoutParsingRequest.viewerDocumentStorageId()))
+                                Layout parsing has finished in %.02f s.
+                                identifiers: %s
+                                %s
+                                Files have been saved with Ids:
+                                Structure: %s
+                                Text: %s
+                                Positions: %s
+                                PageData: %s
+                                Simplified Text: %s
+                                Viewer Doc: %s""",
+                        ((float) (System.currentTimeMillis() - start)) / 1000,
+                        layoutParsingRequest.identifier(),
+                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
+                        layoutParsingRequest.structureFileStorageId(),
+                        layoutParsingRequest.textBlockFileStorageId(),
+                        layoutParsingRequest.positionBlockFileStorageId(),
+                        layoutParsingRequest.pageFileStorageId(),
+                        layoutParsingRequest.simplifiedTextStorageId(),
+                        layoutParsingRequest.viewerDocumentStorageId()))
                .build();

    }


-    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
+    private Document observeBuildDocumentGraph(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument) {

        AtomicReference<Document> documentReference = new AtomicReference<>();

        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry)
                .contextualName("build-document-graph")
-                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument)));
+                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(layoutParsingType, classificationDocument)));

        return documentReference.get();
    }
@@ -200,14 +191,14 @@ public class LayoutParsingPipeline {
    private String buildSemanticNodeCountMessage(int numberOfPages, Map<NodeType, Long> semanticNodeCounts) {

        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
-                             numberOfPages,
-                             semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
-                             semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
-                             semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
-                             semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
-                             semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
-                             semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
-                             semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
+                numberOfPages,
+                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
    }


@@ -218,7 +209,7 @@ public class LayoutParsingPipeline {
                                              ImageServiceResponse imageServiceResponse,
                                              TableServiceResponse tableServiceResponse,
                                              VisualLayoutParsingResponse visualLayoutParsingResponse,
-                                              String identifier) {
+                                              Map<String, String> identifier) {

        PDDocument originDocument = openDocument(originFile);
        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
@@ -263,15 +254,16 @@ public class LayoutParsingPipeline {
            PDRectangle cropbox = pdPage.getCropBox();
            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());

-            List<Cell> emptyTableCells = tableExtractionService.findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());

            ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case REDACT_MANAGER_OLD ->
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells);
                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case DOCSTRUM -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, false);
-                case DOCSTRUM_XY -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, true);
+                case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, true);
+                case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, false);
            };
+
            classificationPage.setCleanRulings(cleanRulings);
            classificationPage.setRotation(rotation);
            classificationPage.setLandscape(isLandscape);
@@ -298,8 +290,10 @@ public class LayoutParsingPipeline {

            tableExtractionService.extractTables(emptyTableCells, classificationPage);

-            if (layoutParsingType == LayoutParsingType.DOCSTRUM || layoutParsingType == LayoutParsingType.DOCSTRUM_XY) {
+            if (layoutParsingType == LayoutParsingType.REDACT_MANAGER) {
                docstrumBlockificationService.combineBlocks(classificationPage);
+            } else if (layoutParsingType == LayoutParsingType.CLARIFYND) {
+                docstrumBlockificationService.mergeIntersectingBlocks(classificationPage.getTextBlocks(), 0, 6.5f);
            }

            buildPageStatistics(classificationPage);
@@ -314,30 +308,22 @@ public class LayoutParsingPipeline {
        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
        log.info("Classify TextBlocks for {}", identifier);
        switch (layoutParsingType) {
-            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG ->
+                    redactManagerClassificationService.classifyDocument(classificationDocument);
            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
-            case DOCSTRUM_XY -> redactManagerClassificationService.classifyDocument(classificationDocument);
-            case DOCSTRUM -> redactManagerClassificationService.classifyDocument(classificationDocument);
+            case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
        }

        log.info("Building Sections for {}", identifier);

-//        if (layoutParsingType == DOCSTRUM || layoutParsingType == DOCSTRUM_XY) {
-//            // Currently for debugging return paragraphs as sections, because there is a merging logic in sectionBuilder
-//            List<ClassificationSection> sections = new ArrayList<>();
-//            for (var page : classificationPages) {
-//                page.getTextBlocks().forEach(block -> {
-//                    block.setPage(page.getPageNumber());
-//                    var section = sectionsBuilderService.buildTextBlock(List.of(block), "a");
-//                    sections.add(section);
-//                });
-//            }
-//            classificationDocument.setSections(sections);
-//        } else {
-        sectionsBuilderService.buildSections(classificationDocument);
-        sectionsBuilderService.addImagesToSections(classificationDocument);
-//        }
+        switch (layoutParsingType) {
+            case CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_PARAGRAPH_DEBUG -> sectionsBuilderService.buildParagraphDebugSections(classificationDocument);
+            default -> {
+                sectionsBuilderService.buildSections(classificationDocument);
+                sectionsBuilderService.addImagesToSections(classificationDocument);
+            }
+        }
+
        return classificationDocument;
    }

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum;

 import java.util.ArrayList;
 import java.util.List;
@@ -6,15 +6,15 @@ import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.LineBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.NearestNeighbourService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.ReadingOrderService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.SpacingService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.ZoneBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.LineBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.NearestNeighbourService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ReadingOrderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.SpacingService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ZoneBuilderService;

 import lombok.RequiredArgsConstructor;

@@ -29,7 +29,7 @@ public class DocstrumSegmentationService {
    private final ReadingOrderService readingOrderService;


-    public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOder) {
+    public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOrder) {

        List<Zone> zones = new ArrayList<>();
        zones.addAll(computeZones(textPositions, TextDirection.ZERO));
@@ -37,7 +37,7 @@ public class DocstrumSegmentationService {
        zones.addAll(computeZones(textPositions, TextDirection.HALF_CIRCLE));
        zones.addAll(computeZones(textPositions, TextDirection.THREE_QUARTER_CIRCLE));

-        return readingOrderService.resolve(zones, xyOder);
+        return readingOrderService.resolve(zones, xyOrder);
    }


@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

 public class AngleFilter {

@@ -7,7 +7,7 @@ public class AngleFilter {


    public AngleFilter(double lowerAngle, double upperAngle) {
-        
+
        this.lowerAngle = lowerAngle < -Math.PI / 2 ? lowerAngle + Math.PI : lowerAngle;
        this.upperAngle = upperAngle >= Math.PI / 2 ? upperAngle - Math.PI : upperAngle;
    }
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

 import java.awt.geom.Rectangle2D;

@@ -45,4 +45,10 @@ public abstract class BoundingBox {
        return bBox.getX() <= contained.getX() + tolerance && bBox.getY() <= contained.getY() + tolerance && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
    }

+
+    public boolean intersectsY(BoundingBox other) {
+
+        return this.getBBox().getMinY() <= other.getBBox().getMaxY() && this.getBBox().getMaxY() >= other.getBBox().getMinY();
+    }
+
 }
@@ -1,20 +1,27 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;

+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;

 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.Setter;

@Data
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Character {

+    @EqualsAndHashCode.Include
    private final double x;
+    @EqualsAndHashCode.Include
    private final double y;
    private final RedTextPosition textPosition;

+    @Setter
    private List<Neighbor> neighbors = new ArrayList<>();


@@ -67,18 +74,12 @@ public class Character {
    }


-    public void setNeighbors(List<Neighbor> neighbors) {
-
-        this.neighbors = neighbors;
-    }
-
-
    public double angle(Character character) {

        if (getX() > character.getX()) {
-            return Math.atan2(getY() - character.getY(), getX() - character.getX());
+            return FastAtan2.fastAtan2(getY() - character.getY(), getX() - character.getX());
        } else {
-            return Math.atan2(character.getY() - getY(), character.getX() - getX());
+            return FastAtan2.fastAtan2(character.getY() - getY(), character.getX() - getX());
        }
    }

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

 public class Histogram {

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
@@ -8,16 +8,22 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

 import lombok.Data;
+import lombok.EqualsAndHashCode;

@Data
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Line extends BoundingBox {

-    private static final double WORD_DISTANCE_MULTIPLIER = 0.2;
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.18;

+    @EqualsAndHashCode.Include
    private final double x0;
+    @EqualsAndHashCode.Include
    private final double y0;

+    @EqualsAndHashCode.Include
    private final double x1;
+    @EqualsAndHashCode.Include
    private final double y1;

    private final double height;
@@ -78,11 +84,7 @@ public class Line extends BoundingBox {

    private double computeHeight() {

-        double sum = 0.0;
-        for (Character component : characters) {
-            sum += component.getHeight();
-        }
-        return sum / characters.size();
+        return characters.stream().map(Character::getHeight).reduce(0d, Double::sum) / characters.size();
    }


@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

 import lombok.Getter;

@@ -6,8 +6,7 @@ public class Neighbor {

    @Getter
    private final double distance;
-    @Getter
-    private final double angle;
+    private Double angle;
    private final Character originCharacter;
    @Getter
    private final Character character;
@@ -16,7 +15,6 @@ public class Neighbor {
    public Neighbor(Character neighbor, Character origin) {

        this.distance = neighbor.distance(origin);
-        this.angle = neighbor.angle(origin);
        this.character = neighbor;
        this.originCharacter = origin;
    }
@@ -33,4 +31,13 @@ public class Neighbor {
        return character.verticalDistance(originCharacter);
    }

+
+    public double getAngle() {
+
+        if (angle != null) {
+            return angle;
+        }
+        return this.character.angle(this.originCharacter);
+    }
+
 }
@@ -0,0 +1,31 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class UnionFind<T> extends org.jgrapht.alg.util.UnionFind<T> {
+
+    public UnionFind(Set<T> elements) {
+
+        super(elements);
+    }
+
+
+    public Collection<Set<T>> getGroups() {
+
+        Map<T, Set<T>> setRep = new LinkedHashMap<>();
+        for (T t : getParentMap().keySet()) {
+            T representative = find(t);
+            if (!setRep.containsKey(representative)) {
+                setRep.put(representative, new LinkedHashSet<>());
+            }
+            setRep.get(representative).add(t);
+        }
+
+        return setRep.values();
+    }
+
+}
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

 import java.awt.geom.Rectangle2D;
 import java.util.Comparator;
@@ -1,15 +1,16 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;

 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;

@Service
 public class LineBuilderService {
@@ -24,7 +25,8 @@ public class LineBuilderService {
        double maxHorizontalDistance = characterSpacing * CHARACTER_SPACING_DISTANCE_MULTIPLIER;
        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_CHARACTER_DISTANCE;

-        DisjointSets<Character> sets = new DisjointSets<>(characters);
+        UnionFind<Character> unionFind = new UnionFind<>(new HashSet<>(characters));
+
        AngleFilter filter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);

        characters.forEach(character -> {
@@ -33,13 +35,13 @@ public class LineBuilderService {
                double y = neighbor.getVerticalDistance() / maxVerticalDistance;
                if (character.getTextPosition().getDir() == neighbor.getCharacter().getTextPosition().getDir() && filter.matches(neighbor) && Math.pow(x, 2) + Math.pow(y,
                        2) <= 1) {
-                    sets.union(character, neighbor.getCharacter());
+                    unionFind.union(character, neighbor.getCharacter());
                }
            });
        });

        List<Line> lines = new ArrayList<>();
-        sets.forEach(group -> {
+        unionFind.getGroups().forEach(group -> {
            List<Character> lineCharacters = new ArrayList<>(group);
            lineCharacters.sort(Comparator.comparingDouble(Character::getX));
            lines.add(new Line(lineCharacters, characterSpacing));
@@ -0,0 +1,36 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+import org.tinspin.index.Index;
+import org.tinspin.index.kdtree.KDTree;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;
+
+@Service
+public class NearestNeighbourService {
+
+    private static final int NUMBER_OF_NEIGHBOURS = 8;
+
+
+    public void findNearestNeighbors(List<Character> characters) {
+        KDTree<Character> kdTree = KDTree.create(2);
+        characters.forEach(c -> kdTree.insert(new double[]{c.getX(), c.getY()}, c));
+
+        for(Character c : characters) {
+            Index.PointIteratorKnn<Character> iterator = kdTree.queryKnn(new double[]{c.getX(), c.getY()}, NUMBER_OF_NEIGHBOURS + 1);
+            // skip the first as this is identity
+            if(iterator.hasNext()) {
+                iterator.next();
+            }
+            while(iterator.hasNext()) {
+                c.getNeighbors().add(new Neighbor(iterator.next().value(), c));
+            }
+        }
+
+    }
+
+}
+
@@ -1,36 +1,59 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;

 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.ListIterator;
+import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;

@Service
 public class ReadingOrderService {

    private static final double THRESHOLD = 5;
+    public static final double MULTI_COLUMN_DETECTION_THRESHOLD = 1.5;


-    public List<Zone> resolve(List<Zone> zones, boolean xyOrder) {
+    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder) {

        if (zones.isEmpty() || zones.size() == 1) {
            return zones;
        }

-        if (xyOrder) {
-//            QuickSort.sort(zones, new ZoneComparator());
-            zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                    .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, 0)));
-            return zones;
+        if (xyReadingOrder) {
+            return resolveSingleColumnReadingOrder(zones);
        }

-        return resolveMultiColumnReadingOder(zones);
+        Map<Long, Integer> histogram = new HashMap<>();
+        for (Zone zone : zones) {
+            long minY = Math.round(zone.getBBox().getMinY());
+            long maxY = Math.round(zone.getBBox().getMaxY());
+            for (long i = minY; i <= maxY; i++) {
+                histogram.put(i, histogram.getOrDefault(i, 0) + 1);
+            }
+        }
+
+        if (histogram.values().stream().mapToInt(Integer::intValue).average().orElse(1) < MULTI_COLUMN_DETECTION_THRESHOLD) {
+            return resolveSingleColumnReadingOrder(zones);
+        } else {
+
+            return resolveMultiColumnReadingOder(zones);
+        }
+
+    }
+
+
+    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones) {
+
+        zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+        return zones;
    }


@@ -75,6 +98,48 @@ public class ReadingOrderService {
        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));

+        List<Zone> leftNotIntersecting = new ArrayList<>();
+        for (Zone leftZone : leftOf) {
+            boolean intersects = false;
+            for (Zone rightZone : rightOf) {
+                if (leftZone.intersectsY(rightZone)) {
+                    intersects = true;
+                    break;
+                }
+                // early stopping
+                if (rightZone.getBBox().getMinY() > leftZone.getBBox().getMaxY()) {
+                    break;
+                }
+            }
+            if (!intersects) {
+                leftNotIntersecting.add(leftZone);
+            }
+        }
+
+        List<Zone> rightNotIntersecting = new ArrayList<>();
+        for (Zone rightZone : rightOf) {
+            boolean intersects = false;
+            for (Zone leftZone : leftOf) {
+                if (rightZone.intersectsY(leftZone)) {
+                    intersects = true;
+                    break;
+                }
+                // early stopping
+                if (leftZone.getBBox().getMinY() > rightZone.getBBox().getMaxY()) {
+                    break;
+                }
+            }
+            if (!intersects) {
+                rightNotIntersecting.add(rightZone);
+            }
+        }
+
+        leftOf.removeAll(leftNotIntersecting);
+        rightOf.removeAll(rightNotIntersecting);
+
+        middle.addAll(leftNotIntersecting);
+        middle.addAll(rightNotIntersecting);
+
        List<Zone> sortedZones = new ArrayList<>();
        sortedZones.addAll(leftOf);
        sortedZones.addAll(rightOf);
@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;

 import java.util.List;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Histogram;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Histogram;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;

@Service
 public class SpacingService {
@@ -1,16 +1,17 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;

 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;

@Service
 public class ZoneBuilderService {
@@ -40,7 +41,7 @@ public class ZoneBuilderService {
        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
        double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;

-        DisjointSets<Line> sets = new DisjointSets<>(lines);
+        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));

        double meanHeight = calculateMeanHeight(lines);

@@ -50,21 +51,21 @@ public class ZoneBuilderService {
                    double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
                    scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));

-                    if (!sets.areTogether(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {
+                    if (!unionFind.inSameSet(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {

                        double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
                        double verticalDistance = outerLine.verticalDistance(innerLine) / scale;

                        if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance //
                                || minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
-                            sets.union(outerLine, innerLine);
+                            unionFind.union(outerLine, innerLine);
                        }
                    }
                }));

        List<Zone> zones = new ArrayList<>();
-        sets.forEach(group -> {
-            zones.add(new Zone(new ArrayList<>(group)));
+        unionFind.getGroups().forEach(group -> {
+            zones.add(mergeLinesInZone(new ArrayList<>(group), characterSpacing, lineSpacing));
        });

        if (zones.size() > MAX_ZONES) {
@@ -99,7 +100,7 @@ public class ZoneBuilderService {
        double minVerticalDistance = 0;
        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE;

-        DisjointSets<Line> sets = new DisjointSets<>(lines);
+        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));

        lines.forEach(outer -> {

@@ -110,7 +111,7 @@ public class ZoneBuilderService {
                    double verticalDistance = outer.verticalDistance(inner);

                    if (horizontalDistance <= maxHorizontalDistance && minVerticalDistance <= verticalDistance && verticalDistance <= maxVerticalDistance) {
-                        sets.union(outer, inner);
+                        unionFind.union(outer, inner);
                    } else if (minVerticalDistance <= verticalDistance && verticalDistance <= maxVerticalDistance && Math.abs(horizontalDistance - Math.min(outer.getLength(),
                            inner.getLength())) < 0.1) {
                        boolean characterOverlap = false;
@@ -127,7 +128,7 @@ public class ZoneBuilderService {
                            }
                        }
                        if (!characterOverlap && overlappingCount <= 2) {
-                            sets.union(outer, inner);
+                            unionFind.union(outer, inner);
                        }
                    }
                }
@@ -135,15 +136,16 @@ public class ZoneBuilderService {
        });

        List<Line> outputZone = new ArrayList<>();
-        for (Set<Line> group : sets) {
-            List<Character> components = new ArrayList<>();
+        for (Set<Line> group : unionFind.getGroups()) {
+            List<Character> characters = new ArrayList<>();
            for (Line line : group) {
-                components.addAll(line.getCharacters());
+                characters.addAll(line.getCharacters());
            }
-            components.sort(Comparator.comparingDouble(Character::getX));
+            characters.sort(Comparator.comparingDouble(Character::getX));

-            outputZone.add(new Line(components, characterSpacing));
+            outputZone.add(new Line(characters, characterSpacing));
        }
+
        return new Zone(outputZone);
    }

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.utils;

 public class DoubleUtils {

@@ -0,0 +1,76 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.utils;
+
+public class FastAtan2 {
+
+    static final private int Size_Ac = 1000;
+    static final private int Size_Ar = Size_Ac + 1;
+    static final private double Pi = (float) Math.PI;
+    static final private double Pi_H = Pi / 2;
+
+    static final private double[] Atan2 = new double[Size_Ar];
+    static final private double[] Atan2_PM = new double[Size_Ar];
+    static final private double[] Atan2_MP = new double[Size_Ar];
+    static final private double[] Atan2_MM = new double[Size_Ar];
+
+    static final private double[] Atan2_R = new double[Size_Ar];
+    static final private double[] Atan2_RPM = new double[Size_Ar];
+    static final private double[] Atan2_RMP = new double[Size_Ar];
+    static final private double[] Atan2_RMM = new double[Size_Ar];
+
+    static {
+        for (int i = 0; i <= Size_Ac; i++) {
+            double d = (double) i / Size_Ac;
+            double x = 1;
+            double y = x * d;
+            double v = Math.atan2(y, x);
+            Atan2[i] = v;
+            Atan2_PM[i] = Pi - v;
+            Atan2_MP[i] = -v;
+            Atan2_MM[i] = -Pi + v;
+
+            Atan2_R[i] = Pi_H - v;
+            Atan2_RPM[i] = Pi_H + v;
+            Atan2_RMP[i] = -Pi_H + v;
+            Atan2_RMM[i] = -Pi_H - v;
+
+        }
+    }
+
+    @SuppressWarnings("ParameterAssignment")
+    static public double fastAtan2(double y, double x) {
+
+        if (y < 0) {
+            if (x < 0) {
+                //(y < x) because == (-y > -x)
+                if (y < x) {
+                    return Atan2_RMM[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_MM[(int) (y / x * Size_Ac)];
+                }
+            } else {
+                y = -y;
+                if (y > x) {
+                    return Atan2_RMP[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_MP[(int) (y / x * Size_Ac)];
+                }
+            }
+        } else {
+            if (x < 0) {
+                x = -x;
+                if (y > x) {
+                    return Atan2_RPM[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_PM[(int) (y / x * Size_Ac)];
+                }
+            } else {
+                if (y > x) {
+                    return Atan2_R[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2[(int) (y / x * Size_Ac)];
+                }
+            }
+        }
+    }
+
+}
@@ -64,9 +64,24 @@ public class Document implements GenericSemanticNode {
    }


+    public List<Header> getHeaders() {
+
+        return streamChildrenOfType(NodeType.HEADER).map(node -> (Header) node)
+                .collect(Collectors.toList());
+    }
+
+
+    public List<Footer> getFooters() {
+
+        return streamChildrenOfType(NodeType.FOOTER).map(node -> (Footer) node)
+                .collect(Collectors.toList());
+    }
+
+
    public Stream<TextBlock> streamTerminalTextBlocksInOrder() {

-        return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getTextBlock);
+        return streamAllNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getTextBlock);
    }


@@ -13,7 +13,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.No
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -40,6 +39,8 @@ public class Image implements GenericSemanticNode {
    boolean transparent;
    Rectangle2D position;

+    TextBlock leafTextBlock;
+
    boolean redaction;
    boolean ignored;
    @Builder.Default
@@ -70,9 +71,7 @@ public class Image implements GenericSemanticNode {
    @Override
    public TextBlock getTextBlock() {

-        return streamAllSubNodes().filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getLeafTextBlock)
-                .collect(new TextBlockCollector());
+        return leafTextBlock;
    }


@@ -98,4 +97,11 @@ public class Image implements GenericSemanticNode {
        return bBoxPerPage;
    }

+
+    @Override
+    public boolean isLeaf() {
+
+        return true;
+    }
+
 }
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import org.apache.pdfbox.text.TextPosition;
-import org.springframework.beans.BeanUtils;

 import com.fasterxml.jackson.annotation.JsonIgnore;

@@ -50,10 +49,14 @@ public class RedTextPosition {
    public static RedTextPosition fromTextPosition(TextPosition textPosition) {

        var pos = new RedTextPosition();
-        BeanUtils.copyProperties(textPosition, pos);
-        pos.setFontName(textPosition.getFont().getName());
-
+        pos.setRotation(textPosition.getRotation());
+        pos.setPageHeight(textPosition.getPageHeight());
+        pos.setPageWidth(textPosition.getPageWidth());
+        pos.setUnicode(textPosition.getUnicode());
+        pos.setDir(textPosition.getDir());
+        pos.setWidthOfSpace(textPosition.getWidthOfSpace());
        pos.setFontSizeInPt(textPosition.getFontSizeInPt());
+        pos.setFontName(textPosition.getFont().getName());

        var position = new float[4];

@@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -24,12 +25,18 @@ import lombok.extern.slf4j.Slf4j;
@Builder
@NoArgsConstructor
@AllArgsConstructor
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TextPositionSequence implements CharSequence {

    public static final int HEIGHT_PADDING = 2;
+
+    @EqualsAndHashCode.Include
    private int page;
+
+    @EqualsAndHashCode.Include
    private List<RedTextPosition> textPositions = new ArrayList<>();

+    @EqualsAndHashCode.Include
    private TextDirection dir;
    private int rotation;
    private float pageHeight;
@@ -201,6 +208,12 @@ public class TextPositionSequence implements CharSequence {
    }


+    public float getTextHeightNoPadding() {
+
+        return textPositions.get(0).getHeightDir();
+    }
+
+
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
@@ -25,6 +25,7 @@ public class BodyTextFrameService {
    private static final float RULING_HEIGHT_THRESHOLD = 0.15f; // multiplied with page height. Header/Footer Rulings must be within that border of the page.
    private static final float RULING_WIDTH_THRESHOLD = 0.75f; // multiplied with page width. Header/Footer Rulings must be at least that wide.

+
    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {

        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
@@ -132,12 +133,7 @@ public class BodyTextFrameService {
                                               boolean landscape,
                                               LayoutParsingType layoutParsingType) {

-        float approximateHeaderLineCount;
-        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
-            approximateHeaderLineCount = 3.3f;
-        } else {
-            approximateHeaderLineCount = 2.9f;
-        }
+        float approximateHeaderLineCount = 2.9f;

        BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();

@@ -155,8 +151,9 @@ public class BodyTextFrameService {
                        continue;
                    }

-                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || MarkedContentUtils.intersects(textBlock,
+                            page.getMarkedContentBboxPerType(),
+                            MarkedContentUtils.FOOTER)) {
                        continue;
                    }

@@ -7,6 +7,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;

+import org.apache.logging.log4j.util.Strings;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -43,7 +44,6 @@ public class SectionsBuilderService {
        for (ClassificationPage page : document.getPages()) {
            List<TextPageBlock> header = new ArrayList<>();
            List<TextPageBlock> footer = new ArrayList<>();
-            List<TextPageBlock> unclassifiedText = new ArrayList<>();
            for (AbstractPageBlock current : page.getTextBlocks()) {

                if (current.getClassification() == null) {
@@ -62,11 +62,6 @@ public class SectionsBuilderService {
                    continue;
                }

-                if (current.getClassification().equals(PageBlockType.OTHER)) {
-                    unclassifiedText.add((TextPageBlock) current);
-                    continue;
-                }
-
                if (prev != null && current.getClassification().isHeadline() && !prev.getClassification().isHeadline() || !document.isHeadlines()) {
                    ClassificationSection chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                    chunkBlock.setHeadline(lastHeadline);
@@ -94,9 +89,6 @@ public class SectionsBuilderService {
            if (!footer.isEmpty()) {
                footers.add(new ClassificationFooter(footer));
            }
-            if (!unclassifiedText.isEmpty()) {
-                unclassifiedTexts.add(new UnclassifiedText(unclassifiedText));
-            }
        }

        ClassificationSection chunkBlock = buildTextBlock(chunkWords, lastHeadline);
@@ -110,6 +102,20 @@ public class SectionsBuilderService {
    }


+    public void buildParagraphDebugSections(ClassificationDocument document) {
+
+        List<ClassificationSection> sections = new ArrayList<>();
+        for (var page : document.getPages()) {
+            page.getTextBlocks().forEach(block -> {
+                block.setPage(page.getPageNumber());
+                var section = buildTextBlock(List.of(block), Strings.EMPTY);
+                sections.add(section);
+            });
+        }
+        document.setSections(sections);
+    }
+
+
    public void addImagesToSections(ClassificationDocument document) {

        Map<Integer, List<ClassificationSection>> sectionMap = new HashMap<>();
@@ -200,6 +206,7 @@ public class SectionsBuilderService {
                            .getX() <= xMax && image.getPosition().getY() >= yMin && image.getPosition().getY() <= yMax) {
                        section.getImages().add(image);
                        image.setAppendedToSection(true);
+                        break;
                    }
                }
                if (!image.isAppendedToSection()) {
@@ -240,7 +247,7 @@ public class SectionsBuilderService {
    }


-    public ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
+    private ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {

        ClassificationSection section = new ClassificationSection();

@@ -1,26 +1,51 @@
 package com.knecon.fforesight.service.layoutparser.processor.services;

 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;

 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+
+import lombok.extern.slf4j.Slf4j;

@Service
+@Slf4j
 public class SimplifiedSectionTextService {

    public SimplifiedText toSimplifiedText(Document document) {

-        List<SimplifiedSectionText> simplifiedSectionTexts = document.getMainSections().stream().map(this::toSimplifiedSectionText).toList();
-        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build();
+        List<SimplifiedSectionText> simplifiedMainSectionsList = document.getMainSections()
+                .stream()
+                .map(this::toSimplifiedSectionText)
+                .toList();
+        List<SimplifiedSectionText> simplifiedHeadersList = document.getHeaders()
+                .stream()
+                .map(this::toSimplifiedSectionText)
+                .toList();
+        List<SimplifiedSectionText> simplifiedFootersList = document.getFooters()
+                .stream()
+                .map(this::toSimplifiedSectionText)
+                .toList();
+        List<SimplifiedSectionText> simplifiedText = Stream.of(simplifiedMainSectionsList, simplifiedHeadersList, simplifiedFootersList)
+                .flatMap(List::stream)
+                .collect(Collectors.toList());
+        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedText).build();
    }


-    private SimplifiedSectionText toSimplifiedSectionText(Section section) {
+    private SimplifiedSectionText toSimplifiedSectionText(SemanticNode section) {

-        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
+        return SimplifiedSectionText.builder()
+                .sectionNumber(section.getTreeId()
+                                       .get(0))
+                .text(section.getTextBlock().getSearchText())
+                .build();
    }
+
 }
@@ -2,29 +2,28 @@ package com.knecon.fforesight.service.layoutparser.processor.services.blockifica

 import static java.util.stream.Collectors.toSet;

-import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.HashSet;
 import java.util.List;
 import java.util.ListIterator;
-import java.util.Set;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.docstrum.DocstrumSegmentationService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
-import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
-import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionSequenceComparator;

 import lombok.RequiredArgsConstructor;

@@ -38,21 +37,24 @@ public class DocstrumBlockificationService {
    static final float THRESHOLD = 1f;


-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Cell> cells, boolean xyOder) {
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Cell> cells, boolean xyOrder) {

-        // Underlined or strikethrough are also in rulings but we dont want to split blocks with them so we use cells.
-        List<Ruling> usedHorizonalRulings = new ArrayList<>();
-        List<Ruling> usedVerticalRulings = new ArrayList<>();
+        CleanRulings usedRulings = RectangleTransformations.extractRulings(cells);

-        cells.forEach(cell -> {
-            usedHorizonalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y), new Point2D.Float(cell.x + cell.width, cell.y)));
-            usedHorizonalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y + cell.height), new Point2D.Float(cell.x + cell.width, cell.y + cell.height)));
-            usedVerticalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y), new Point2D.Float(cell.x, cell.y + cell.height)));
-            usedVerticalRulings.add(new Ruling(new Point2D.Float(cell.x + cell.width, cell.y), new Point2D.Float(cell.x + cell.width, cell.y + cell.height)));
-        });
+        var zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder);
+        var pageBlocks = toAbstractPageBlocks(zones, usedRulings.getHorizontal(), usedRulings.getVertical(), xyOrder);
+
+        var classificationPage = new ClassificationPage(pageBlocks);
+
+        mergeIntersectingBlocks(classificationPage.getTextBlocks(), 0, 0);
+
+        return classificationPage;
+    }
+
+
+    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones, List<Ruling> horizontalRulings, List<Ruling> verticalRulings, boolean xyOrder) {

        List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
-        var zones = docstrumSegmentationService.segmentPage(textPositions, xyOder);
        zones.forEach(zone -> {

            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
@@ -62,83 +64,62 @@ public class DocstrumBlockificationService {
                });
            });

-            abstractPageBlocks.addAll(splitZonesAtRulings(textPositionSequences, usedHorizonalRulings, usedVerticalRulings));
-//            abstractPageBlocks.add(buildTextBlock(textPositionSequences, 0));
+            abstractPageBlocks.addAll(splitZonesAtRulings(textPositionSequences, horizontalRulings, verticalRulings));
        });

-        return new ClassificationPage(abstractPageBlocks);
+        if (xyOrder) {
+            abstractPageBlocks.sort(Comparator.comparing(AbstractPageBlock::getMinY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(AbstractPageBlock::getMinX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+            abstractPageBlocks.sort(new Comparator<AbstractPageBlock>() {
+                @Override
+                public int compare(AbstractPageBlock o1, AbstractPageBlock o2) {
+
+                    return Math.abs(o1.getMinY() - o2.getMinY()) < 5 && o1.getMinX() < o2.getMinX() == true ? -1 : 0;
+                }
+            });
+        }
+
+        return abstractPageBlocks;
    }


    public void combineBlocks(ClassificationPage page) {

-        mergeZones(page.getTextBlocks());
-
        TextPageBlock previous = new TextPageBlock();
        ListIterator<AbstractPageBlock> itty = page.getTextBlocks().listIterator();
        while (itty.hasNext()) {
+
            AbstractPageBlock block = itty.next();
            if (block instanceof TablePageBlock) {
+                previous = new TextPageBlock();
                continue;
            }
            TextPageBlock current = (TextPageBlock) block;

            if (previous != null && !previous.getSequences().isEmpty()) {

-                if (current.getDir() == previous.getDir() //
-                        && previous.getNumberOfLines() >= 2 && current.getNumberOfLines() >= 2 //
-                        && previous.intersectsY(current) //
-                        && !hasBetween(current, previous, page.getTextBlocks()) //
-                        && numberOfYIntersections(current, previous, page.getTextBlocks()) == 0) {
-
-                    previous.getSequences().addAll(current.getSequences());
-                    previous = buildTextBlock(previous.getSequences(), 0);
-                    previous.setToDuplicate(true);
-                    itty.remove();
-                    itty.previous();
-                    itty.set(previous);
-                    itty.next();
+                if (current.getDir() != previous.getDir()) {
+                    previous = current;
                    continue;
                }

-                if (current.getDir() == previous.getDir() && (previous.almostIntersects(current, 0, 0))) {
-
-                    previous.getSequences().addAll(current.getSequences());
-                    boolean toDuplicate = previous.isToDuplicate();
-                    previous = buildTextBlock(previous.getSequences(), 0);
-                    previous.setToDuplicate(toDuplicate);
-                    itty.remove();
-                    itty.previous();
-                    itty.set(previous);
-                    itty.next();
+                if (areTheOnlyTwoBlocksOnHeightsWithBothMoreThanTwoLines(previous, current, page)) {
+                    previous = combineBlocksAndResetIterator(previous, current, itty, true);
                    continue;
                }

-                if (current.getDir() == previous.getDir() //
-                        && (Math.abs(previous.getMaxY() - current.getMaxY()) < THRESHOLD || Math.abs(previous.getMinY() - current.getMinY()) < THRESHOLD) //
-                        && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
-                        && !hasBetween(current, previous, page.getTextBlocks()) && numberOfYIntersections(current, previous, page.getTextBlocks()) <= 4) {
-
-                    previous.getSequences().addAll(current.getSequences());
-                    previous = buildTextBlock(previous.getSequences(), 0);
-                    itty.remove();
-                    itty.previous();
-                    itty.set(previous);
-                    itty.next();
+                if (previous.almostIntersects(current, 0, 0)) {
+                    previous = combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
                    continue;
                }

-                if (current.getDir() == previous.getDir() //
-                        && current.intersectsY(previous) //
-                        && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
-                        && !hasBetween(current, previous, page.getTextBlocks()) //
-                        && numberOfYIntersections(current, previous, page.getTextBlocks()) <= 0) {
-                    previous.getSequences().addAll(current.getSequences());
-                    previous = buildTextBlock(previous.getSequences(), 0);
-                    itty.remove();
-                    itty.previous();
-                    itty.set(previous);
-                    itty.next();
+                if (isSameTopOrBottomWithPreviousHavingMaxTwoLinesAndCurrentThanOneAndMax4OtherBlocksOnHeight(previous, current, page)) {
+                    previous = combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
+                    continue;
+                }
+
+                if (isOnlyIntersectingYAndOnelineOrPrevoiusTwoLines(previous, current, page)) {
+                    previous = combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
                    continue;
                }

@@ -146,7 +127,46 @@ public class DocstrumBlockificationService {
            previous = current;
        }

-        mergeZones(page.getTextBlocks());
+        mergeIntersectingBlocks(page.getTextBlocks(), 0, 6.5f);
+    }
+
+
+    private boolean isOnlyIntersectingYAndOnelineOrPrevoiusTwoLines(TextPageBlock previous, TextPageBlock current, ClassificationPage page) {
+
+        return current.intersectsY(previous) //
+                && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
+                && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) <= 0;
+    }
+
+
+    private boolean isSameTopOrBottomWithPreviousHavingMaxTwoLinesAndCurrentThanOneAndMax4OtherBlocksOnHeight(TextPageBlock previous,
+                                                                                                              TextPageBlock current,
+                                                                                                              ClassificationPage page) {
+
+        return (Math.abs(previous.getMaxY() - current.getMaxY()) < THRESHOLD || Math.abs(previous.getMinY() - current.getMinY()) < THRESHOLD) //
+                && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() >= 2 && current.getNumberOfLines() == 1) //
+                && !hasBetween(current, previous, page.getTextBlocks()) && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) <= 4;
+    }
+
+
+    private boolean areTheOnlyTwoBlocksOnHeightsWithBothMoreThanTwoLines(TextPageBlock previous, TextPageBlock current, ClassificationPage page) {
+
+        return previous.getNumberOfLines() >= 2 && current.getNumberOfLines() >= 2 //
+                && previous.intersectsY(current) //
+                && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) == 0;
+    }
+
+
+    private TextPageBlock combineBlocksAndResetIterator(TextPageBlock previous, TextPageBlock current, ListIterator<AbstractPageBlock> itty, boolean toDuplicate) {
+
+        previous.getSequences().addAll(current.getSequences());
+        previous = buildTextBlock(previous.getSequences(), 0);
+        previous.setToDuplicate(toDuplicate);
+        itty.remove();
+        itty.previous();
+        itty.set(previous);
+        itty.next();
+        return previous;
    }


@@ -167,7 +187,7 @@ public class DocstrumBlockificationService {
    }


-    private int numberOfYIntersections(TextPageBlock block, TextPageBlock other, List<AbstractPageBlock> allBlocks) {
+    private int numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(TextPageBlock block, TextPageBlock other, List<AbstractPageBlock> allBlocks) {

        double minY = Math.min(block.getMinY(), other.getMinY());
        double maxY = Math.min(block.getMaxY(), other.getMaxY());
@@ -188,51 +208,52 @@ public class DocstrumBlockificationService {
    }


-    private void mergeZones(List<AbstractPageBlock> zones) {
+    public void mergeIntersectingBlocks(List<AbstractPageBlock> blocks, float xThreshold, float yThreshold) {

-        ListIterator<AbstractPageBlock> itty = zones.listIterator();
-        Set<AbstractPageBlock> toRemove = new HashSet<>();
+        ListIterator<AbstractPageBlock> itty = blocks.listIterator();
        while (itty.hasNext()) {
            AbstractPageBlock block = itty.next();
+            if(block == null){
+                continue;
+            }
            if (block instanceof TablePageBlock) {
                continue;
            }

            TextPageBlock current = (TextPageBlock) block;

-            if (current.isToDuplicate()) {
-                continue;
-            }
+            for (int i = 0; i < blocks.size(); i++) {

-            for (int i = 0; i < zones.size(); i++) {
-
-                if (toRemove.contains(zones.get(i))) {
+                if(blocks.get(i) == null){
                    continue;
                }
-                if (zones.get(i) == current) {
+                if (blocks.get(i) == current) {
                    continue;
                }
-                if (zones.get(i) instanceof TablePageBlock) {
+                if (blocks.get(i) instanceof TablePageBlock) {
                    continue;
                }

-                TextPageBlock inner = (TextPageBlock) zones.get(i);
+                TextPageBlock inner = (TextPageBlock) blocks.get(i);

-                if (inner.isToDuplicate()) {
-                    continue;
-                }
-
-                if (current.getDir() == inner.getDir() && current.almostIntersects(inner, 0, 0)) {
+                if (current.getDir() == inner.getDir() && current.almostIntersects(inner, yThreshold, xThreshold)) {

+                    boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
                    current.getSequences().addAll(inner.getSequences());
-                    QuickSort.sort(current.getSequences(), new TextPositionSequenceComparator());
                    current = buildTextBlock(current.getSequences(), 0);
-                    toRemove.add(inner);
+
+                    current.setToDuplicate(toDuplicate);
+                    blocks.set(i, null);
                    itty.set(current);
                }
            }
        }
-        zones.removeAll(toRemove);
+        var blocksIterator = blocks.iterator();
+        while(blocksIterator.hasNext()){
+            if(blocksIterator.next() == null){
+                blocksIterator.remove();
+            }
+        }
    }


@@ -13,10 +13,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;

@SuppressWarnings("all")
@@ -31,12 +34,12 @@ public class RedactManagerBlockificationService {
     * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
     * Rulings (TablePageBlock lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
     *
-     * @param textPositions         The words of a page.
-     * @param horizontalRulingLines Horizontal table lines.
-     * @param verticalRulingLines   Vertical table lines.
+     * @param textPositions The words of a page.
     * @return Page object that contains the Textblock and text statistics.
     */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Cell> cells) {
+
+        CleanRulings usedRulings = RectangleTransformations.extractRulings(cells);

        int indexOnPage = 0;
        List<TextPositionSequence> chunkWords = new ArrayList<>();
@@ -54,7 +57,7 @@ public class RedactManagerBlockificationService {
            boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
            boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
-            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
+            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, usedRulings.getHorizontal(), usedRulings.getVertical());
            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());

            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
@@ -1,330 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.blockification;
-
-
-// TODO: figure out, why this fails the build
-// import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;
-
-@Service
-@SuppressWarnings("all")
-public class TaasBlockificationService {
-
-    private static final float THRESHOLD = 1f;
-    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f; // multiplied with text height
-    private static final float INTERSECTS_Y_THRESHOLD = 4;// 2 * HEIGHT_PADDING // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
-    private static final int X_GAP_SPLIT_CONSTANT = 50;
-    public static final int X_ALIGNMENT_THRESHOLD = 1;
-    public static final int NEGATIVE_X_GAP_THRESHOLD = -5;
-
-    private Pattern listIdentifier = Pattern.compile("^(?:(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)]))|\\uF0B7", Pattern.CASE_INSENSITIVE);
-
-
-    /**
-     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
-     * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
-     * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
-     *
-     * @param textPositions         The words of a page.
-     * @param horizontalRulingLines Horizontal table lines.
-     * @param verticalRulingLines   Vertical table lines.
-     * @return ClassificationPage object that contains the Textblock and text statistics.
-     */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
-
-        List<TextPageBlock> classificationTextBlocks = constructFineGranularTextPageBlocks(textPositions, horizontalRulingLines, verticalRulingLines);
-        classificationTextBlocks = mergeTextPageBlocksAligningX(classificationTextBlocks);
-        classificationTextBlocks = mergeIntersectingTextBlocksUntilConvergence(classificationTextBlocks);
-
-        return new ClassificationPage(new ArrayList<>(classificationTextBlocks.stream().map(classificationTextBlock -> (AbstractPageBlock) classificationTextBlock).toList()));
-    }
-
-
-    private List<TextPageBlock> mergeIntersectingTextBlocksUntilConvergence(List<TextPageBlock> classificationTextBlocks) {
-
-        int currentSize = classificationTextBlocks.size();
-        while (true) {
-            classificationTextBlocks = mergeTextPageBlocksAlmostIntersecting(classificationTextBlocks);
-            if (classificationTextBlocks.size() == currentSize) {
-                break;
-            }
-            currentSize = classificationTextBlocks.size();
-        }
-        return classificationTextBlocks;
-    }
-
-
-    private List<TextPageBlock> mergeTextPageBlocksAligningX(List<TextPageBlock> classificationTextBlocks) {
-
-        if (classificationTextBlocks.isEmpty()) {
-            return new ArrayList<>();
-        }
-        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
-        List<TextPageBlock> currentTextBlocksToMerge = new LinkedList<>();
-        textBlocksToMerge.add(currentTextBlocksToMerge);
-        TextPageBlock previousTextBlock = null;
-        Float lastLineGap = null;
-        for (TextPageBlock currentTextBlock : classificationTextBlocks) {
-            if (previousTextBlock == null) {
-                currentTextBlocksToMerge.add(currentTextBlock);
-                previousTextBlock = currentTextBlock;
-                continue;
-            }
-
-
-            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
-            boolean isListIdentifier = listIdentifierPattern.find();
-
-            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-
-            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
-//            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-
-            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
-            boolean alignsXLeft = Math.abs(currentTextBlock.getPdfMinX() - previousTextBlock.getPdfMinX()) < X_ALIGNMENT_THRESHOLD;
-//            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < yGap;
-            if (yGap && sameFont && !isListIdentifier) {
-                currentTextBlocksToMerge.add(currentTextBlock);
-
-            } else {
-                currentTextBlocksToMerge = new LinkedList<>();
-                currentTextBlocksToMerge.add(currentTextBlock);
-                textBlocksToMerge.add(currentTextBlocksToMerge);
-            }
-            previousTextBlock = currentTextBlock;
-        }
-        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
-    }
-
-
-    private List<TextPageBlock> mergeTextPageBlocksAlmostIntersecting(List<TextPageBlock> textPageBlocks) {
-
-        Set<TextPageBlock> alreadyMerged = new HashSet<>();
-        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
-        for (TextPageBlock textPageBlock : textPageBlocks) {
-            if (alreadyMerged.contains(textPageBlock)) {
-                continue;
-            }
-            alreadyMerged.add(textPageBlock);
-            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
-                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
-                    .toList());
-        }
-        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
-    }
-
-
-    private void assignOrientations(List<TextPageBlock> classificationTextBlocks) {
-
-        Iterator<TextPageBlock> itty = classificationTextBlocks.iterator();
-
-        TextPageBlock previousLeft = null;
-        TextPageBlock previousRight = null;
-        while (itty.hasNext()) {
-            TextPageBlock block = (TextPageBlock) itty.next();
-
-            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
-                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
-                    previousLeft.add(block);
-                    itty.remove();
-                    continue;
-                }
-            }
-
-            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
-                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
-                    previousRight.add(block);
-                    itty.remove();
-                    continue;
-                }
-            }
-
-            if (block.getOrientation().equals(Orientation.LEFT)) {
-                previousLeft = block;
-            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
-                previousRight = block;
-            }
-        }
-
-        itty = classificationTextBlocks.iterator();
-        TextPageBlock previous = null;
-        while (itty.hasNext()) {
-            TextPageBlock block = (TextPageBlock) itty.next();
-
-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(
-                    block.getMaxY(),
-                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
-                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
-                previous.add(block);
-                itty.remove();
-                continue;
-            }
-
-            previous = block;
-        }
-    }
-
-
-    private List<TextPageBlock> constructFineGranularTextPageBlocks(List<TextPositionSequence> textPositions,
-                                                                    List<Ruling> horizontalRulingLines,
-                                                                    List<Ruling> verticalRulingLines) {
-
-        int indexOnPage = 0;
-        List<TextPositionSequence> wordClusterToCombine = new ArrayList<>();
-        List<TextPageBlock> classificationTextBlocks = new ArrayList<>();
-
-        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
-        TextPositionSequence prev = null;
-        // TODO: make static final constant
-
-
-        boolean wasSplitted = false;
-        Float splitX1 = null;
-        for (TextPositionSequence word : textPositions) {
-
-            Matcher listIdentifierPattern = listIdentifier.matcher(word.toString());
-
-            boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-            boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());
-            boolean positiveXGapInline = prev != null && maxX + X_GAP_SPLIT_CONSTANT < word.getMinXDirAdj() && sameLine;
-            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < NEGATIVE_X_GAP_THRESHOLD;
-            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
-            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
-            boolean splitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
-            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean fontChange = prev != null && (!word.getFont().equals(prev.getFont()) || !word.getFontStyle()
-                    .equals(prev.getFontStyle()) || word.getFontSize() != prev.getFontSize());
-            boolean newline = prev != null && Math.abs(word.getMinYDirAdj() - prev.getMinYDirAdj()) > word.getHeight();
-            boolean isListIdentifier = listIdentifierPattern.matches();
-
-            if (prev != null && (prev.isParagraphStart() || negativeXGap || positiveXGapInline || yGap || startFromTop || splitByRuling || (newline && (fontChange || isListIdentifier)))) {
-//            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
-
-                Orientation prevOrientation = null;
-                if (!classificationTextBlocks.isEmpty()) {
-                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - X_ALIGNMENT_THRESHOLD).getOrientation();
-                }
-
-                TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);
-
-                classificationTextBlocks.add(classificationTextBlock);
-                wordClusterToCombine = new ArrayList<>();
-
-                if (positiveXGapInline && !splitByRuling) {
-                    wasSplitted = true;
-                    classificationTextBlock.setOrientation(Orientation.LEFT);
-                    splitX1 = word.getMinXDirAdj();
-                } else if (newLineAfterSplit && !splitByRuling) {
-                    wasSplitted = false;
-                    classificationTextBlock.setOrientation(Orientation.RIGHT);
-                    splitX1 = null;
-                } else if (prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (yGap || !startFromTop || !positiveXGapInline || !newLineAfterSplit || !splitByRuling)) {
-                    classificationTextBlock.setOrientation(Orientation.LEFT);
-                }
-
-                minX = 1000;
-                maxX = 0;
-                minY = 1000;
-                maxY = 0;
-                prev = null;
-            }
-
-            wordClusterToCombine.add(word);
-
-            prev = word;
-            if (word.getMinXDirAdj() < minX) {
-                minX = word.getMinXDirAdj();
-            }
-            if (word.getMaxXDirAdj() > maxX) {
-                maxX = word.getMaxXDirAdj();
-            }
-            if (word.getMinYDirAdj() < minY) {
-                minY = word.getMinYDirAdj();
-            }
-            if (word.getMaxYDirAdj() > maxY) {
-                maxY = word.getMaxYDirAdj();
-            }
-        }
-
-        TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);
-        if (classificationTextBlock != null) {
-            classificationTextBlocks.add(classificationTextBlock);
-        }
-        return classificationTextBlocks;
-    }
-
-
-    private boolean equalsWithThreshold(float f1, float f2) {
-
-        return Math.abs(f1 - f2) < THRESHOLD;
-    }
-
-
-    private boolean isSplitByRuling(float minX,
-                                    float minY,
-                                    float maxX,
-                                    float maxY,
-                                    TextPositionSequence word,
-                                    List<Ruling> horizontalRulingLines,
-                                    List<Ruling> verticalRulingLines) {
-
-        return isSplitByRuling(maxX,
-                minY,
-                word.getMinXDirAdj(),
-                word.getMinYDirAdj(),
-                verticalRulingLines,
-                word.getDir().getDegrees(),
-                word.getPageWidth(),
-                word.getPageHeight()) //
-                || isSplitByRuling(minX,
-                minY,
-                word.getMinXDirAdj(),
-                word.getMaxYDirAdj(),
-                horizontalRulingLines,
-                word.getDir().getDegrees(),
-                word.getPageWidth(),
-                word.getPageHeight()) //
-                || isSplitByRuling(maxX,
-                minY,
-                word.getMinXDirAdj(),
-                word.getMinYDirAdj(),
-                horizontalRulingLines,
-                word.getDir().getDegrees(),
-                word.getPageWidth(),
-                word.getPageHeight()) //
-                || isSplitByRuling(minX,
-                minY,
-                word.getMinXDirAdj(),
-                word.getMaxYDirAdj(),
-                verticalRulingLines,
-                word.getDir().getDegrees(),
-                word.getPageWidth(),
-                word.getPageHeight()); //
-    }
-
-
-    private boolean isSplitByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines, float dir, float pageWidth, float pageHeight) {
-
-        for (Ruling ruling : rulingLines) {
-            var line = RulingTextDirAdjustUtil.convertToDirAdj(ruling, dir, pageWidth, pageHeight);
-            if (line.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-}
-
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -11,7 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -20,26 +19,21 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
-public class TaasClassificationService {
-
-    private final BodyTextFrameService bodyTextFrameService;
-
+public class ClarifyndClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
-
            classifyPage(page, document, headlineFontSizes);
        }
    }


-    public void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {

        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextPageBlock) {
@@ -49,20 +43,25 @@ public class TaasClassificationService {
    }


-    public void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {

        var bodyTextFrame = page.getBodyTextFrame();

        if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
-            textBlock.setClassification(PageBlockType.HEADER);
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
-            textBlock.setClassification(PageBlockType.FOOTER);
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
                .size() == 1)) {
@@ -107,8 +106,9 @@ public class TaasClassificationService {
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
            textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
        } else {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
        }
    }

 }
+
@@ -1,194 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.util.AbstractSet;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-public class DisjointSets<E> implements Iterable<Set<E>> {
-
-    private final Map<E, Entry<E>> map = new HashMap<>();
-
-
-    public DisjointSets(Collection<? extends E> collection) {
-
-        for (E element : collection) {
-            map.put(element, new Entry<E>(element));
-        }
-    }
-
-
-    public boolean areTogether(E e1, E e2) {
-
-        return map.get(e1).findRepresentative().equals(map.get(e2).findRepresentative());
-    }
-
-
-    public void union(E e1, E e2) {
-
-        Entry<E> r1 = map.get(e1).findRepresentative();
-        Entry<E> r2 = map.get(e2).findRepresentative();
-        if (!r1.equals(r2)) {
-            if (r1.size <= r2.size) {
-                r2.mergeWith(r1);
-            } else {
-                r1.mergeWith(r2);
-            }
-        }
-    }
-
-
-    @Override
-    public Iterator<Set<E>> iterator() {
-
-        return new Iterator<>() {
-
-            private final Iterator<Entry<E>> iterator = map.values().iterator();
-            private Entry<E> nextRepresentative;
-
-            {
-                findNextRepresentative();
-            }
-
-            @Override
-            public boolean hasNext() {
-
-                return nextRepresentative != null;
-            }
-
-
-            @Override
-            public Set<E> next() {
-
-                if (nextRepresentative == null) {
-                    throw new NoSuchElementException();
-                }
-                Set<E> result = nextRepresentative.asSet();
-                findNextRepresentative();
-                return result;
-            }
-
-
-            private void findNextRepresentative() {
-
-                while (iterator.hasNext()) {
-                    Entry<E> candidate = iterator.next();
-                    if (candidate.isRepresentative()) {
-                        nextRepresentative = candidate;
-                        return;
-                    }
-                }
-                nextRepresentative = null;
-            }
-
-
-            @Override
-            public void remove() {
-
-                throw new UnsupportedOperationException();
-            }
-
-        };
-    }
-
-
-    private static class Entry<E> {
-
-        private int size = 1;
-        private final E value;
-        private Entry<E> parent = this;
-        private Entry<E> next;
-        private Entry<E> last = this;
-
-
-        Entry(E value) {
-
-            this.value = value;
-        }
-
-
-        void mergeWith(Entry<E> otherRepresentative) {
-
-            size += otherRepresentative.size;
-            last.next = otherRepresentative;
-            last = otherRepresentative.last;
-            otherRepresentative.parent = this;
-        }
-
-
-        Entry<E> findRepresentative() {
-
-            Entry<E> representative = parent;
-            while (!representative.parent.equals(representative)) {
-                representative = representative.parent;
-            }
-            for (Entry<E> entry = this; !entry.equals(representative); ) {
-                Entry<E> nextEntry = entry.parent;
-                entry.parent = representative;
-                entry = nextEntry;
-            }
-            return representative;
-        }
-
-
-        boolean isRepresentative() {
-
-            return parent.equals(this);
-        }
-
-
-        Set<E> asSet() {
-
-            return new AbstractSet<E>() {
-
-                @Override
-                public Iterator<E> iterator() {
-
-                    return new Iterator<E>() {
-
-                        private Entry<E> nextEntry = findRepresentative();
-
-
-                        @Override
-                        public boolean hasNext() {
-
-                            return nextEntry != null;
-                        }
-
-
-                        @Override
-                        public E next() {
-
-                            if (nextEntry == null) {
-                                throw new NoSuchElementException();
-                            }
-                            E result = nextEntry.value;
-                            nextEntry = nextEntry.next;
-                            return result;
-                        }
-
-
-                        @Override
-                        public void remove() {
-
-                            throw new UnsupportedOperationException();
-                        }
-
-                    };
-                }
-
-
-                @Override
-                public int size() {
-
-                    return findRepresentative().size;
-                }
-            };
-        }
-
-    }
-
-}
@@ -1,78 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
-
-@Service
-public class NearestNeighbourService {
-
-    private static final int NUMBER_OF_NEIGHBOURS = 8;
-    private static final double STEP = 16.0;
-
-
-    public void findNearestNeighbors(List<Character> characters) {
-
-        if (characters.isEmpty()) {
-            return;
-        }
-
-        characters.sort(Comparator.comparingDouble(Character::getX));
-
-        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
-        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
-            maxNeighborCount = characters.size() - 1;
-        }
-
-        for (int i = 0; i < characters.size(); i++) {
-
-            List<Neighbor> candidates = new ArrayList<>();
-
-            int start = i;
-            int end = i + 1;
-
-            double distance = Double.POSITIVE_INFINITY;
-
-            for (double searchDistance = 0; searchDistance < distance; ) {
-
-                searchDistance += STEP;
-                boolean newCandidatesFound = false;
-
-                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
-                    start--;
-                    candidates.add(new Neighbor(characters.get(start), characters.get(i)));
-                    clearLeastDistant(candidates, maxNeighborCount);
-                    newCandidatesFound = true;
-                }
-
-                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
-                    candidates.add(new Neighbor(characters.get(end), characters.get(i)));
-                    clearLeastDistant(candidates, maxNeighborCount);
-                    end++;
-                    newCandidatesFound = true;
-                }
-
-                if (newCandidatesFound && candidates.size() >= maxNeighborCount) {
-                    distance = candidates.get(maxNeighborCount - 1).getDistance();
-                }
-            }
-            clearLeastDistant(candidates, maxNeighborCount);
-            characters.get(i).setNeighbors(new ArrayList<>(candidates));
-        }
-    }
-
-
-    private void clearLeastDistant(List<Neighbor> candidates, int maxNeighborCount) {
-
-        if (candidates.size() > maxNeighborCount) {
-            candidates.sort(Comparator.comparingDouble(Neighbor::getDistance));
-            candidates.remove(candidates.remove(candidates.size() - 1));
-        }
-    }
-
-}
@@ -16,6 +16,7 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
@@ -48,14 +49,19 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class DocumentGraphFactory {

-    public Document buildDocumentGraph(ClassificationDocument document) {
+    public Document buildDocumentGraph(LayoutParsingType layoutParsingType, ClassificationDocument document) {

        Document documentGraph = new Document();
        Context context = new Context(documentGraph);

-        document.getPages().forEach(context::buildAndAddPageWithCounter);
-        document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.getImages().add(image));
-        addSections(document, context);
+        document.getPages()
+                .forEach(context::buildAndAddPageWithCounter);
+        document.getSections()
+                .stream()
+                .flatMap(section -> section.getImages()
+                        .stream())
+                .forEach(image -> context.getImages().add(image));
+        addSections(layoutParsingType, document, context, documentGraph);
        addHeaderAndFooterToEachPage(document, context);

        documentGraph.setNumberOfPages(context.pages.size());
@@ -66,9 +72,10 @@ public class DocumentGraphFactory {
    }


-    private void addSections(ClassificationDocument document, Context context) {
+    private void addSections(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument, Context context, Document document) {

-        document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getNonEmptyPageBlocks(), section.getImages(), context));
+        classificationDocument.getSections()
+                .forEach(section -> SectionNodeFactory.addSection(layoutParsingType, null, section.getNonEmptyPageBlocks(), section.getImages(), context, document));
    }


@@ -78,11 +85,14 @@ public class DocumentGraphFactory {

        GenericSemanticNode node;
        if (originalTextBlock.isHeadline()) {
-            node = Headline.builder().documentTree(context.getDocumentTree()).build();
+            node = Headline.builder().documentTree(context.getDocumentTree())
+                    .build();
        } else if (originalTextBlock.isToDuplicate()) {
-            node = DuplicatedParagraph.builder().documentTree(context.getDocumentTree()).build();
+            node = DuplicatedParagraph.builder().documentTree(context.getDocumentTree())
+                    .build();
        } else {
-            node = Paragraph.builder().documentTree(context.getDocumentTree()).build();
+            node = Paragraph.builder().documentTree(context.getDocumentTree())
+                    .build();
        }

        page.getMainBody().add(node);
@@ -90,12 +100,14 @@ public class DocumentGraphFactory {
        List<TextPageBlock> textBlocks = new ArrayList<>();
        textBlocks.add(originalTextBlock);
        textBlocks.addAll(textBlocksToMerge);
+
        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);

        if (node instanceof DuplicatedParagraph duplicatedParagraph) {
            AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock(textBlocks.stream()
-                    .flatMap(tb -> tb.getSequences().stream())
-                    .collect(Collectors.toList()), node, context, page);
+                                                                                                      .flatMap(tb -> tb.getSequences()
+                                                                                                              .stream())
+                                                                                                      .collect(Collectors.toList()), node, context, page);
            duplicatedParagraph.setUnsortedLeafTextBlock(unsortedTextBlock);
        }

@@ -105,25 +117,29 @@ public class DocumentGraphFactory {
    }


-    public void addImage(Section section, ClassifiedImage image, Context context) {
+    public void addImage(GenericSemanticNode parent, ClassifiedImage image, Context context) {
+
+        Image imageNode = createImage(image, context);
+        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parent, imageNode);
+        imageNode.setTreeId(treeId);
+        imageNode.setLeafTextBlock(context.textBlockFactory.emptyTextBlock(parent, context, context.getPage(image.getPage())));
+    }
+
+
+    private Image createImage(ClassifiedImage image, Context context) {

        Rectangle2D position = image.getPosition();
        Page page = context.getPage(image.getPage());
-        var imageBuilder = Image.builder()
+        Image imageNode = Image.builder()
                .id(IdBuilder.buildId(Set.of(page), List.of(position)))
                .imageType(image.getImageType())
                .position(position)
                .transparent(image.isHasTransparency())
                .page(page)
-                .documentTree(context.getDocumentTree());
-        if (image.isSourceByAi()) {
-            imageBuilder.engines(new HashSet<>(Set.of(LayoutEngine.AI)));
-        }
-        Image imageNode = imageBuilder.build();
+                .documentTree(context.getDocumentTree())
+                .build();
        page.getMainBody().add(imageNode);
-
-        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode);
-        imageNode.setTreeId(treeId);
+        return imageNode;
    }


@@ -162,11 +178,12 @@ public class DocumentGraphFactory {
    private void addFooter(List<TextPageBlock> textBlocks, Context context) {

        Page page = context.getPage(textBlocks.get(0).getPage());
-        Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build();
+        Footer footer = Footer.builder().documentTree(context.getDocumentTree())
+                .build();
        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
-                footer,
-                context,
-                page);
+                                                                                  footer,
+                                                                                  context,
+                                                                                  page);
        List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer);
        footer.setTreeId(tocId);
        footer.setLeafTextBlock(textBlock);
@@ -177,7 +194,8 @@ public class DocumentGraphFactory {
    public void addHeader(List<TextPageBlock> textBlocks, Context context) {

        Page page = context.getPage(textBlocks.get(0).getPage());
-        Header header = Header.builder().documentTree(context.getDocumentTree()).build();
+        Header header = Header.builder().documentTree(context.getDocumentTree())
+                .build();
        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, 0, page);
        List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header);
        header.setTreeId(tocId);
@@ -189,7 +207,8 @@ public class DocumentGraphFactory {
    private void addEmptyFooter(int pageIndex, Context context) {

        Page page = context.getPage(pageIndex);
-        Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build();
+        Footer footer = Footer.builder().documentTree(context.getDocumentTree())
+                .build();
        AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page);
        List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer);
        footer.setTreeId(tocId);
@@ -201,7 +220,8 @@ public class DocumentGraphFactory {
    private void addEmptyHeader(int pageIndex, Context context) {

        Page page = context.getPage(pageIndex);
-        Header header = Header.builder().documentTree(context.getDocumentTree()).build();
+        Header header = Header.builder().documentTree(context.getDocumentTree())
+                .build();
        AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
        List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header);
        header.setTreeId(tocId);
@@ -4,13 +4,16 @@ import static java.lang.String.format;
 import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.groupingBy;

+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
@@ -24,27 +27,52 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class SectionNodeFactory {

-    public void addSection(GenericSemanticNode parentNode, List<AbstractPageBlock> pageBlocks, List<ClassifiedImage> images, DocumentGraphFactory.Context context) {
+    public void addSection(LayoutParsingType layoutParsingType,
+                           GenericSemanticNode parentNode,
+                           List<AbstractPageBlock> pageBlocks,
+                           List<ClassifiedImage> images,
+                           DocumentGraphFactory.Context context,
+                           Document document) {
+
+        // This is for the case where we have images on a page without any text/footer/header.
+        // The pageBlocks list is empty, but we still need to add those images to the document.
+        if (!images.isEmpty() && pageBlocks.isEmpty()) {
+            images.stream()
+                    .distinct()
+                    .forEach(image -> DocumentGraphFactory.addImage(document, image, context));
+            return;
+        }

        if (pageBlocks.isEmpty()) {
            return;
        }
-        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
-        Section section = Section.builder().documentTree(context.getDocumentTree()).build();
+
+        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream()
+                .collect(groupingBy(AbstractPageBlock::getPage));
+        Section section = Section.builder().documentTree(context.getDocumentTree())
+                .build();

        context.getSections().add(section);
-        blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));
+        blocksPerPage.keySet()
+                .forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));

        section.setTreeId(getTreeId(parentNode, context, section));

-        addFirstHeadlineDirectlyToSection(pageBlocks, context, section);
+        addFirstHeadlineDirectlyToSection(layoutParsingType, pageBlocks, context, section, document);
        if (containsTablesAndTextBlocks(pageBlocks)) {
-            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(section, subSectionPageBlocks, emptyList(), context));
+            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(layoutParsingType,
+                                                                                                  section,
+                                                                                                  subSectionPageBlocks,
+                                                                                                  emptyList(),
+                                                                                                  context,
+                                                                                                  document));
        } else {
-            addTablesAndParagraphsAndHeadlinesToSection(pageBlocks, context, section);
+            addTablesAndParagraphsAndHeadlinesToSection(layoutParsingType, pageBlocks, context, section, document);
        }

-        images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(section, image, context));
+        images.stream()
+                .distinct()
+                .forEach(image -> DocumentGraphFactory.addImage(section, image, context));
    }


@@ -58,16 +86,24 @@ public class SectionNodeFactory {
    }


-    private void addFirstHeadlineDirectlyToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
+    private void addFirstHeadlineDirectlyToSection(LayoutParsingType layoutParsingType,
+                                                   List<AbstractPageBlock> pageBlocks,
+                                                   DocumentGraphFactory.Context context,
+                                                   Section section,
+                                                   Document document) {

        if (pageBlocks.get(0).isHeadline()) {
-            addTablesAndParagraphsAndHeadlinesToSection(List.of(pageBlocks.get(0)), context, section);
+            addTablesAndParagraphsAndHeadlinesToSection(layoutParsingType, List.of(pageBlocks.get(0)), context, section, document);
            pageBlocks.remove(0);
        }
    }


-    private void addTablesAndParagraphsAndHeadlinesToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
+    private void addTablesAndParagraphsAndHeadlinesToSection(LayoutParsingType layoutParsingType,
+                                                             List<AbstractPageBlock> pageBlocks,
+                                                             DocumentGraphFactory.Context context,
+                                                             Section section,
+                                                             Document document) {

        Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
        List<AbstractPageBlock> remainingBlocks = new LinkedList<>(pageBlocks);
@@ -80,13 +116,23 @@ public class SectionNodeFactory {
            remainingBlocks.removeAll(alreadyMerged);

            if (abstractPageBlock instanceof TextPageBlock) {
-                List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY((TextPageBlock) abstractPageBlock, remainingBlocks);
-                alreadyMerged.addAll(textBlocks);
-                DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
+
+                switch (layoutParsingType) {
+                    case REDACT_MANAGER, CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> {
+                        alreadyMerged.add(abstractPageBlock);
+                        remainingBlocks.remove(abstractPageBlock);
+                        DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, new ArrayList<>());
+                    }
+                    default -> {
+                        List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY((TextPageBlock) abstractPageBlock, remainingBlocks);
+                        alreadyMerged.addAll(textBlocks);
+                        DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
+                    }
+                }
            } else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
                List<TablePageBlock> tablesToMerge = TableMergingUtility.findConsecutiveTablesWithSameColCountAndSameHeaders(tablePageBlock, remainingBlocks);
                alreadyMerged.addAll(tablesToMerge);
-                TableNodeFactory.addTable(section, tablesToMerge, context);
+                TableNodeFactory.addTable(layoutParsingType, section, tablesToMerge, context, document);
            } else {
                throw new RuntimeException(format("Unhandled AbstractPageBlockType %s!", abstractPageBlock.getClass()));
            }
@@ -96,7 +142,9 @@ public class SectionNodeFactory {

    private boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {

-        return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
+        return pageBlocks.stream()
+                       .anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
+                       .anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
    }


@@ -112,7 +160,9 @@ public class SectionNodeFactory {

        List<List<AbstractPageBlock>> splitList = splitIntoCoherentList(pageBlocks);
        movePrecedingHeadlineToTableList(splitList);
-        return splitList.stream().filter(list -> !list.isEmpty()).toList();
+        return splitList.stream()
+                .filter(list -> !list.isEmpty())
+                .toList();
    }


@@ -133,7 +183,8 @@ public class SectionNodeFactory {

    private boolean listIsTablesOnly(List<AbstractPageBlock> abstractPageBlocks) {

-        return abstractPageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock);
+        return abstractPageBlocks.stream()
+                .allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock);
    }


@@ -7,16 +7,18 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import lombok.experimental.UtilityClass;
@@ -27,23 +29,30 @@ public class TableNodeFactory {
    public final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;


-    public void addTable(GenericSemanticNode parentNode, List<TablePageBlock> tablesToMerge, DocumentGraphFactory.Context context) {
+    public void addTable(LayoutParsingType layoutParsingType,
+                         GenericSemanticNode parentNode,
+                         List<TablePageBlock> tablesToMerge,
+                         DocumentGraphFactory.Context context,
+                         Document document) {

        setPageNumberInCells(tablesToMerge);
-        Set<Page> pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet());
-        List<List<Cell>> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList();
+        Set<Page> pages = tablesToMerge.stream()
+                .map(AbstractPageBlock::getPage)
+                .map(context::getPage)
+                .collect(Collectors.toSet());
+        List<List<Cell>> mergedRows = tablesToMerge.stream()
+                .map(TablePageBlock::getRows)
+                .flatMap(Collection::stream)
+                .toList();

-        Table table = Table.builder()
-                .documentTree(context.getDocumentTree())
-                .numberOfCols(mergedRows.isEmpty() ? 0 : mergedRows.get(0).size())
-                .numberOfRows(mergedRows.size())
+        Table table = Table.builder().documentTree(context.getDocumentTree()).numberOfCols(mergedRows.isEmpty() ? 0 : mergedRows.get(0).size()).numberOfRows(mergedRows.size())
                .build();

        pages.forEach(page -> addTableToPage(page, parentNode, table));

        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table);
        table.setTreeId(treeId);
-        addTableCells(mergedRows, table, context);
+        addTableCells(layoutParsingType, mergedRows, table, context, document);

        ifTableHasNoHeadersSetFirstRowAsHeaders(table);
    }
@@ -63,7 +72,8 @@ public class TableNodeFactory {

    private void setPageNumberInTextBlocksWithPageNumberSetTo0(TablePageBlock table, Cell cell) {

-        cell.getTextBlocks().stream()//
+        cell.getTextBlocks()
+                .stream()//
                .filter(tb -> tb.getPage() == 0)//
                .forEach(tb -> tb.setPage(table.getPage()));
    }
@@ -82,28 +92,44 @@ public class TableNodeFactory {

    private void ifTableHasNoHeadersSetFirstRowAsHeaders(Table table) {

-        if (table.streamHeaders().findAny().isEmpty()) {
-            table.streamRow(0).forEach(tableCellNode -> tableCellNode.setHeader(true));
+        if (table.streamHeaders()
+                .findAny().isEmpty()) {
+            table.streamRow(0)
+                    .forEach(tableCellNode -> tableCellNode.setHeader(true));
        }
    }


-    private void addTableCells(List<List<Cell>> rows, Table table, DocumentGraphFactory.Context context) {
+    private void addTableCells(LayoutParsingType layoutParsingType, List<List<Cell>> rows, Table table, DocumentGraphFactory.Context context, Document document) {

        for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
            for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, table, context);
+                addTableCell(layoutParsingType,
+                             rows.get(rowIndex)
+                                     .get(colIndex),
+                             rowIndex,
+                             colIndex,
+                             table,
+                             context,
+                             document);
            }
        }
    }


    @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong
-    private void addTableCell(Cell cell, int rowIndex, int colIndex, Table tableNode, DocumentGraphFactory.Context context) {
+    private void addTableCell(LayoutParsingType layoutParsingType,
+                              Cell cell,
+                              int rowIndex,
+                              int colIndex,
+                              Table tableNode,
+                              DocumentGraphFactory.Context context,
+                              Document document) {

        Page page = context.getPage(cell.getPageNumber());

-        TableCell tableCell = TableCell.builder().documentTree(context.getDocumentTree()).row(rowIndex).col(colIndex).header(cell.isHeaderCell()).bBox(cell.getBounds2D()).build();
+        TableCell tableCell = TableCell.builder().documentTree(context.getDocumentTree()).row(rowIndex).col(colIndex).header(cell.isHeaderCell()).bBox(cell.getBounds2D())
+                .build();
        page.getMainBody().add(tableCell);

        List<Integer> treeId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell);
@@ -113,16 +139,27 @@ public class TableNodeFactory {
        if (cell.getTextBlocks().isEmpty()) {
            tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page));
        } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page);
+            textBlock = context.getTextBlockFactory()
+                    .buildAtomicTextBlock(cell.getTextBlocks()
+                                                  .get(0).getSequences(), tableCell, context, page);
            tableCell.setLeafTextBlock(textBlock);
        } else if (firstTextBlockIsHeadline(cell)) {
-            SectionNodeFactory.addSection(tableCell, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
+            SectionNodeFactory.addSection(layoutParsingType,
+                                          tableCell,
+                                          cell.getTextBlocks()
+                                                  .stream()
+                                                  .map(tb -> (AbstractPageBlock) tb)
+                                                  .toList(),
+                                          emptyList(),
+                                          context,
+                                          document);
        } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
            tableCell.setLeafTextBlock(textBlock);
        } else {
-            cell.getTextBlocks().forEach(tb -> DocumentGraphFactory.addParagraphOrHeadline(tableCell, tb, context, emptyList()));
+            cell.getTextBlocks()
+                    .forEach(tb -> DocumentGraphFactory.addParagraphOrHeadline(tableCell, tb, context, emptyList()));
        }
    }

@@ -135,7 +172,8 @@ public class TableNodeFactory {

    private boolean firstTextBlockIsHeadline(Cell cell) {

-        return cell.getTextBlocks().get(0).isHeadline();
+        return cell.getTextBlocks()
+                .get(0).isHeadline();
    }

 }
@@ -129,11 +129,13 @@ public class LayoutGridService {
            return;
        }
        for (Page page : table.getPages()) {
+
            Optional<Integer> optionalFirstRowOnPage = table.streamCol(0).filter(tableCell -> tableCell.isOnPage(page.getNumber())).map(TableCell::getRow).findFirst();
            if (optionalFirstRowOnPage.isEmpty()) {
                continue;
            }
            int firstRowOnPage = optionalFirstRowOnPage.get();
+
            Stream<Double> xStream = switch (page.getRotation()) {
                case 90 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinX);
                case 180 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMaxX);
@@ -142,6 +144,7 @@ public class LayoutGridService {
            };
            List<Double> xs = xStream.collect(Collectors.toList());
            xs.remove(0);
+
            Stream<Double> yStream = switch (page.getRotation()) {
                case 90 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMinY);
                case 180 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinY);
@@ -151,7 +154,7 @@ public class LayoutGridService {
            List<Double> ys = yStream.collect(Collectors.toList());
            ys.remove(0);

-            Rectangle2D tableBBox = table.getBBox().get(table.getFirstPage());
+            Rectangle2D tableBBox = table.getBBox().get(page);
            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredLines();
            xs.forEach(x -> {
                Line2D line = new Line2D.Double(new Point2D.Double(x, tableBBox.getMaxY()), new Point2D.Double(x, tableBBox.getMinY()));
@@ -2,9 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;

 import static java.lang.String.format;

-import java.awt.geom.Area;
+import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -20,6 +21,8 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;

 import lombok.AllArgsConstructor;
 import lombok.NoArgsConstructor;
@@ -40,11 +43,10 @@ public class RectangleTransformations {

    public static double calculateIntersectedArea(Rectangle2D r1, Rectangle2D r2) {

-        Area a1 = new Area(r1);
-        Area a2 = new Area(r2);
-        a1.intersect(a2);
-        Rectangle2D intersection = a1.getBounds2D();
-        return intersection.getWidth() * intersection.getHeight();
+        double xOverlap = Math.max(0, Math.min(r1.getMaxX(), r2.getMaxX()) - Math.max(r1.getMinX(), r2.getMinX()));
+        double yOverlap = Math.max(0, Math.min(r1.getMaxY(), r2.getMaxY()) - Math.max(r1.getY(), r2.getY()));
+
+        return xOverlap * yOverlap;
    }


@@ -149,7 +151,27 @@ public class RectangleTransformations {
                previousRectangle = currentRectangle;
            }
        }
-        return rectangleListsWithGaps.stream().map(RectangleTransformations::rectangle2DBBox).toList();
+        return rectangleListsWithGaps.stream()
+                .map(RectangleTransformations::rectangle2DBBox)
+                .toList();
+    }
+
+
+    public static CleanRulings extractRulings(List<? extends Rectangle2D.Float> rectangles) {
+
+        // Underlined or strikethrough are also in rulings but we dont want to split blocks with them so we use cells.
+        List<Ruling> horizontalRulings = new ArrayList<>();
+        List<Ruling> verticalRulings = new ArrayList<>();
+
+        rectangles.forEach(rectangle -> {
+            horizontalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y), new Point2D.Float(rectangle.x + rectangle.width, rectangle.y)));
+            horizontalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y + rectangle.height),
+                                             new Point2D.Float(rectangle.x + rectangle.width, rectangle.y + rectangle.height)));
+            verticalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y), new Point2D.Float(rectangle.x, rectangle.y + rectangle.height)));
+            verticalRulings.add(new Ruling(new Point2D.Float(rectangle.x + rectangle.width, rectangle.y),
+                                           new Point2D.Float(rectangle.x + rectangle.width, rectangle.y + rectangle.height)));
+        });
+        return CleanRulings.builder().vertical(verticalRulings).horizontal(horizontalRulings).build();
    }


@@ -28,15 +28,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPosit
 *
 * @author Ben Litchfield
 */
-public class TextPositionSequenceComparator implements Comparator<TextPositionSequence>
-{
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
+
    @Override
-    public int compare(TextPositionSequence pos1, TextPositionSequence pos2)
-    {
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
        // only compare text that is in the same direction
        int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
-        if (cmp1 != 0)
-        {
+        if (cmp1 != 0) {
            return cmp1;
        }

@@ -48,25 +46,19 @@ public class TextPositionSequenceComparator implements Comparator<TextPositionSe
        float pos2YBottom = pos2.getMaxYDirAdj();

        // note that the coordinates have been adjusted so 0,0 is in upper left
-        float pos1YTop = pos1YBottom - pos1.getTextHeight();
-        float pos2YTop = pos2YBottom - pos2.getTextHeight();
+        float pos1YTop = pos1YBottom - pos1.getTextHeightNoPadding();
+        float pos2YTop = pos2YBottom - pos2.getTextHeightNoPadding();

        float yDifference = Math.abs(pos1YBottom - pos2YBottom);

        // we will do a simple tolerance comparison
-        if (yDifference < .1 ||
-                pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
-                pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
-        {
+        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
            return Float.compare(x1, x2);
-        }
-        else if (pos1YBottom < pos2YBottom)
-        {
+        } else if (pos1YBottom < pos2YBottom) {
            return -1;
-        }
-        else
-        {
+        } else {
            return 1;
        }
    }
+
 }
@@ -11,14 +11,13 @@ import org.springframework.context.annotation.Import;
 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
-import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

 import io.micrometer.observation.ObservationRegistry;
 import io.micrometer.observation.aop.ObservedAspect;

@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
-@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
+@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

@@ -1,6 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.server.queue;

 import java.util.Arrays;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;

 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
@@ -28,6 +31,7 @@ public class MessageHandler {
    private final LayoutParsingPipeline layoutParsingPipeline;
    private final ObjectMapper objectMapper;
    private final RabbitTemplate rabbitTemplate;
+    private final static String X_PIPELINE_PREFIX = "X-PIPE-";


    @RabbitHandler
@@ -37,7 +41,7 @@ public class MessageHandler {

        LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);

-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS) && layoutParsingRequest.researchDocumentStorageId() == null) {
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.CLARIFYND) && layoutParsingRequest.researchDocumentStorageId() == null) {
            throw new IllegalArgumentException("ResearchDocumentDataStorageId is null!");
        }
        log.info("Layout parsing request received {}", layoutParsingRequest.identifier());
@@ -46,14 +50,24 @@ public class MessageHandler {
                    layoutParsingRequest.identifier()));
        }
        LayoutParsingFinishedEvent layoutParsingFinishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent);
+        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent, message);
    }


-    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
+    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent, Message message) {

        Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent, m -> {
+            var forwardHeaders = message.getMessageProperties()
+                    .getHeaders()
+                    .entrySet()
+                    .stream()
+                    .filter(e -> e.getKey().toUpperCase(Locale.ROOT).startsWith(X_PIPELINE_PREFIX))
+                    .collect(Collectors.toMap(Map.Entry::getKey,
+                            Map.Entry::getValue));
+            m.getMessageProperties().getHeaders().putAll(forwardHeaders);
+            return m;
+        });
    }

 }
@@ -1,37 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server.queue;
-
-import org.springframework.amqp.core.Queue;
-import org.springframework.amqp.core.QueueBuilder;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
-
-import lombok.RequiredArgsConstructor;
-
-@Configuration
-@RequiredArgsConstructor
-public class MessagingConfiguration {
-
-    @Bean
-    public Queue layoutparsingRequestQueue() {
-
-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingResponseQueue() {
-
-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingDLQ() {
-
-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
-    }
-
-}
@@ -11,6 +11,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;

 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -48,12 +49,13 @@ public class BdrJsonBuildTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(File file) {

-        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
-                file,
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
-                file.toString()));
+        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.CLARIFYND,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.CLARIFYND,
+                        file,
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        Map.of("file",file.toString())));
    }


@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.server;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;

@@ -95,12 +96,13 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

-        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                pdfFileResource.getFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
-                filePath));
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                        pdfFileResource.getFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        Map.of("file",filePath)));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -28,7 +28,20 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
        prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
+        Arrays.stream(finishedEvent.message().split("\n"))
+                .forEach(log::info);
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testLayoutParserEndToEnd_RED_8747() {
+
+        prepareStorage("files/SinglePages/MergedEntities.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
+        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
+        Arrays.stream(finishedEvent.message().split("\n"))
+                .forEach(log::info);
    }

 }
@@ -0,0 +1,78 @@
+package com.knecon.fforesight.service.layoutparser.server;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+
+import io.micrometer.observation.Observation;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class SimplifiedTextServiceTest
+        extends AbstractTest {
+
+    @Autowired
+    private LayoutParsingPipeline layoutParsingPipeline;
+    @Autowired
+    private SimplifiedSectionTextService simplifiedSectionTextService;
+
+
+    @Test
+    @SneakyThrows
+    public void testSearchIndexing() {
+
+        File file = new ClassPathResource("files/headerFooterTest3Pages.pdf").getFile();
+        String footerExample = "Footer to search for ";
+        String headerExample ="Header to search for ";
+        Document document = buildGraph(file);
+        SimplifiedText simplifiedText = simplifiedSectionTextService.toSimplifiedText(document);
+        List<SimplifiedSectionText> sectionTexts = simplifiedText.getSectionTexts();
+        assertThat(sectionTexts.stream().filter(section -> section.getText().equals(footerExample)).collect(Collectors.toList()).size()).isGreaterThan(0);
+        assertThat(sectionTexts.stream().filter(section -> section.getText().equals(headerExample)).collect(Collectors.toList()).size()).isGreaterThan(0);
+
+
+
+    }
+
+    @SneakyThrows
+    protected Document buildGraph(File file) {
+
+        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
+                                                       layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                                                                                         file,
+                                                                                         new ImageServiceResponse(),
+                                                                                         new TableServiceResponse(),
+                                                                                         new VisualLayoutParsingResponse(),
+                                                                                         Map.of("file",file.toString())));
+    }
+
+
+
+}
@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;

 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -55,12 +56,13 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
    @SneakyThrows
    private void writeJsons(Path filename) {

-        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
-                filename.toFile().toString()));
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                        filename.toFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        Map.of("file",filename.toFile().toString())));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.server.graph;

 import java.io.File;
 import java.nio.file.Path;
+import java.util.Map;

 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -26,15 +27,15 @@ public class ViewerDocumentTest extends BuildDocumentTest {
    @SneakyThrows
    public void testViewerDocument() {

-        String fileName = "files/new/270 rotated text on non rotated pages.pdf";
+        String fileName = "files/new/ScrambledTextAfterSorting.pdf";
        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";

        var documentFile = new ClassPathResource(fileName).getFile();
        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
        LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);

-        Document document = buildGraph(fileName, LayoutParsingType.DOCSTRUM_XY);
        long start = System.currentTimeMillis();
+        Document document = buildGraph(fileName, LayoutParsingType.DOCUMINE);
        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
        System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
    }
@@ -58,10 +59,10 @@ public class ViewerDocumentTest extends BuildDocumentTest {
                new ImageServiceResponse(),
                tableResponse,
                new VisualLayoutParsingResponse(),
-                Path.of(fileName).getFileName().toFile().toString());
+                Map.of("file", Path.of(fileName).getFileName().toFile().toString()));
        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
        LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
-        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+        Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE, classificationDocument);

        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
    }
@@ -56,12 +56,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @SneakyThrows
    public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {

-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                                                                                          originDocument,
-                                                                                          new ImageServiceResponse(),
-                                                                                          tableServiceResponse,
-                                                                                          new VisualLayoutParsingResponse(),
-                                                                                          "document");
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                originDocument,
+                new ImageServiceResponse(),
+                tableServiceResponse,
+                new VisualLayoutParsingResponse(),
+                Map.of("file","document"));

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -112,16 +112,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        var tables = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();

        // Quality of the table parsing is not good, because the file is rotated at scanning.
        // We only asset that the table border is not the page border.
@@ -143,12 +135,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        imageServiceResponse.getData()
                .forEach(imageMetadata -> images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
                        .add(new ClassifiedImage(new Rectangle2D.Double(imageMetadata.getPosition().getX1(),
-                                                                        imageMetadata.getPosition().getY1(),
-                                                                        imageMetadata.getGeometry().getWidth(),
-                                                                        imageMetadata.getGeometry().getHeight()),
-                                                 ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)),
-                                                 imageMetadata.isAlpha(),
-                                                 imageMetadata.getPosition().getPageNumber())));
+                                imageMetadata.getPosition().getY1(),
+                                imageMetadata.getGeometry().getWidth(),
+                                imageMetadata.getGeometry().getHeight()),
+                                ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)),
+                                imageMetadata.isAlpha(),
+                                imageMetadata.getPosition().getPageNumber())));

        System.out.println("object");
    }
@@ -160,22 +152,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock table = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
        assertThat(table.getColCount()).isEqualTo(6);
        assertThat(table.getRowCount()).isEqualTo(13);
-        assertThat(table.getRows()
-                           .stream()
-                           .mapToInt(List::size).sum()).isEqualTo(6 * 13);
+        assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
    }


@@ -185,37 +166,15 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
        assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
        assertThat(secondTable.getColCount()).isEqualTo(8);
        assertThat(secondTable.getRowCount()).isEqualTo(2);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(0)
-                .stream()
-                .map(Collections::singletonList)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
    }


@@ -225,37 +184,15 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
        assertThat(firstTable.getColCount()).isEqualTo(9);
        assertThat(firstTable.getRowCount()).isEqualTo(5);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
        assertThat(secondTable.getColCount()).isEqualTo(9);
        assertThat(secondTable.getRowCount()).isEqualTo(6);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(firstTable.getRowCount() - 1)
-                .stream()
-                .map(Cell::getHeaderCells)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
    }


@@ -265,37 +202,15 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
        assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
        assertThat(secondTable.getColCount()).isEqualTo(8);
        assertThat(secondTable.getRowCount()).isEqualTo(6);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(0)
-                .stream()
-                .map(Collections::singletonList)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
    }


@@ -345,30 +260,29 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        validateTable(document, 0, 8, 8, 0, 0);

        List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
-                                                                "Author, date",
-                                                                "Study title",
-                                                                "Analytical method Author, date, No.",
-                                                                "Technique, LOQ of the method, validated working range",
-                                                                "Method meets analytical validation criteria",
-                                                                "Remarks (in case validation criteria are not met)",
-                                                                "Acceptability of the method"),
-                                                  Arrays.asList(
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
-                                                  Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
-                                                                "Evans P.G. 2001 TMJ4569B, VV-323245",
-                                                                "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
-                                                                "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
-                                                                "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
-                                                                "Y",
-                                                                "N/A",
-                                                                "Y"));
+                        "Author, date",
+                        "Study title",
+                        "Analytical method Author, date, No.",
+                        "Technique, LOQ of the method, validated working range",
+                        "Method meets analytical validation criteria",
+                        "Remarks (in case validation criteria are not met)",
+                        "Acceptability of the method"),
+                Arrays.asList("Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
+                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
+                        "Evans P.G. 2001 TMJ4569B, VV-323245",
+                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
+                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
+                        "Y",
+                        "N/A",
+                        "Y"));

        validateTable(document, 0, values);

@@ -753,15 +667,24 @@ public class PdfSegmentationServiceTest extends AbstractTest {

    }

+    @Test
+    public void testMergedEntities_Page26() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/MergedEntities.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 6, 6, 5, 0);
+
+    }
+

    @SneakyThrows
    private void toHtml(ClassificationDocument document, String filename) {

-        var tables = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList();
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
        StringBuilder sb = new StringBuilder();

        int currentPage = 1;
@@ -782,19 +705,9 @@ public class PdfSegmentationServiceTest extends AbstractTest {

    private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {

-        TablePageBlock table = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(tableIndex);
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
        List<List<Cell>> rows = table.getRows();
-        int emptyCellsFoundFound = rows.stream()
-                .flatMap(List::stream)
-                .toList()
-                .stream()
-                .filter(f -> f.toString().isEmpty())
-                .toList().size();
+        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().isEmpty()).toList().size();

        for (List<Cell> row : table.getRows()) {
            row.forEach(r -> System.out.println(r.toString()));
@@ -809,20 +722,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {

    private void validateTable(ClassificationDocument document, int tableIndex, List<List<String>> values) {

-        TablePageBlock table = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(tableIndex);
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
        List<List<Cell>> rows = table.getRows();

-        List<Cell> rowsFlattened = rows.stream()
-                .flatMap(List::stream)
-                .toList();
-        List<String> valuesFlattened = values.stream()
-                .flatMap(List::stream)
-                .toList();
+        List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();
+        List<String> valuesFlattened = values.stream().flatMap(List::stream).toList();

        for (int i = 0; i < valuesFlattened.size(); i++) {
            Cell cell = rowsFlattened.get(i);
@@ -835,11 +739,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

    private void validateTableSize(ClassificationDocument document, int tableSize) {

-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .toList().size()).isEqualTo(tableSize);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().size()).isEqualTo(tableSize);

    }

@@ -21,7 +21,7 @@ class BodyTextFrameServiceTest extends BuildDocumentTest {

        String filename = "files/211.pdf";
        String outputFilename = "/tmp/" + Path.of(filename).getFileName() + "_MAINBODY.pdf";
-        ClassificationDocument document = parseLayout(filename, LayoutParsingType.TAAS);
+        ClassificationDocument document = parseLayout(filename, LayoutParsingType.CLARIFYND);
        PdfDraw.drawRectanglesPerPage(filename,
                document.getPages().stream().map(page -> List.of(RectangleTransformations.toRectangle2D(page.getBodyTextFrame()))).toList(),
                outputFilename);
@@ -7,6 +7,7 @@ import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;

 import org.apache.pdfbox.Loader;
@@ -74,7 +75,7 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings()));
        }
        var cleanRulings = cleanRulingsPerPage.stream().map(CleanRulings::getVertical).collect(Collectors.toList());
-        PdfDraw.drawLinesPerPage(fileName,  cleanRulings, lineFileName);
+        PdfDraw.drawLinesPerPage(fileName, cleanRulings, lineFileName);

    }

@@ -99,18 +100,20 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
    @SneakyThrows
    private void writeJsons(Path filename) {

-        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
-                filename.toFile().toString()));
-        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
-                filename.toFile().toString()));
+        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                        filename.toFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        Map.of("file",filename.toFile().toString())));
+        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                        filename.toFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        Map.of("file",filename.toFile().toString())));
        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure())) {
@@ -20,7 +20,6 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
-import org.xmlunit.builder.Input;

 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.storage.commons.service.StorageService;
@@ -68,7 +67,7 @@ public abstract class AbstractTest {
    protected LayoutParsingRequest buildStandardLayoutParsingRequest() {

        return LayoutParsingRequest.builder()
-                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER_OLD)
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
@@ -99,7 +98,7 @@ public abstract class AbstractTest {
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(String file) {

-        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json","visual_layout_parsing_response/empty.json");
+        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json", "visual_layout_parsing_response/empty.json");
    }


@@ -107,7 +106,7 @@ public abstract class AbstractTest {
    protected LayoutParsingRequest prepareStorage(InputStream fileInputStream) {

        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileInputStream);
-        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
    }


@@ -140,6 +139,7 @@ public abstract class AbstractTest {
        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream());
    }

+
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile, String visualLayoutParsingResponseFile) {

@@ -148,9 +148,13 @@ public abstract class AbstractTest {
        ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
        ClassPathResource visualLayoutParsingResponseResource = new ClassPathResource(visualLayoutParsingResponseFile);

-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream(), visualLayoutParsingResponseResource.getInputStream());
+        return prepareStorage(pdfFileResource.getInputStream(),
+                cvServiceResponseFileResource.getInputStream(),
+                imageInfoFileResource.getInputStream(),
+                visualLayoutParsingResponseResource.getInputStream());
    }

+
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream) {

@@ -158,18 +162,22 @@ public abstract class AbstractTest {
        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);

-        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
    }

+
    @SneakyThrows
-    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream, InputStream visualLayoutParsingResponseFileStream) {
+    protected LayoutParsingRequest prepareStorage(InputStream fileStream,
+                                                  InputStream cvServiceResponseFileStream,
+                                                  InputStream imageInfoStream,
+                                                  InputStream visualLayoutParsingResponseFileStream) {

        storageService.storeObject(TenantContext.getTenantId(), IMAGE_FILE_ID, imageInfoStream);
        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);
-        storageService.storeObject(TenantContext.getTenantId(),VISUAL_LAYOUT_FILE,visualLayoutParsingResponseFileStream );
+        storageService.storeObject(TenantContext.getTenantId(), VISUAL_LAYOUT_FILE, visualLayoutParsingResponseFileStream);

-        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
    }


@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

 import java.io.File;
+import java.util.Map;

 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
@@ -31,14 +32,14 @@ public abstract class BuildDocumentTest extends AbstractTest {
                layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
                new TableServiceResponse(),
                new VisualLayoutParsingResponse(),
-                filename);
+                Map.of("file",filename));
    }


    @SneakyThrows
    protected Document buildGraph(String filename) {

-        return buildGraph(filename, LayoutParsingType.REDACT_MANAGER);
+        return buildGraph(filename, LayoutParsingType.REDACT_MANAGER_OLD);
    }


@@ -51,7 +52,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
            prepareStorage(filename);
        }

-        return DocumentGraphFactory.buildDocumentGraph(parseLayout(filename, layoutParsingType));
+        return DocumentGraphFactory.buildDocumentGraph(layoutParsingType, parseLayout(filename, layoutParsingType));
    }

 }
@@ -1,6 +1,6 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.2.2"
+    id("io.freefair.lombok") version "8.4"
 }

 description = "Library for adding/removing layers in the viewer document"
@@ -17,6 +17,7 @@ import java.util.stream.Collectors;

 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdfwriter.compress.CompressParameters;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -285,7 +286,7 @@ public class ViewerDocumentService {

        Observation.createNotStarted("ViewerDocumentService", registry).contextualName("incremental-save").observe(() -> {
            try (var out = new FileOutputStream(outputFile)) {
-                pdDocument.save(out);
+                pdDocument.save(out, CompressParameters.NO_COMPRESSION);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
Author	SHA1	Message	Date
Kilian Schüttler	d38d023485	Merge branch 'RED-7384' into 'main' Red 7384 See merge request fforesight/layout-parser!140	2024-04-23 12:13:21 +02:00
Kilian Schüttler	c1afe9b11f	Red 7384	2024-04-23 12:13:19 +02:00
Dominique Eifländer	c947d552d2	Merge branch 'RED-8995-fp' into 'main' RED-8995: unclassified text might be missing from document data See merge request fforesight/layout-parser!135	2024-04-19 09:21:50 +02:00
Kilian Schuettler	f256f9b30f	RED-8995: unclassified text might be missing from document data * treat TablePageBlock.OTHER like PARAGRAPH (no special treatment)	2024-04-18 17:42:34 +02:00
Yannik Hampe	6167e3fb57	Merge branch 'RED-8402' into 'main' RED-8402: Header and footer are not indexed / searched See merge request fforesight/layout-parser!134	2024-04-18 15:08:00 +02:00
yhampe	a78fb0244a	Merge remote-tracking branch 'origin/RED-8402' into RED-8402	2024-04-18 14:39:10 +02:00
yhampe	8099a00bb6	RED-8402: Header and footer are not indexed / searched added unit test and file	2024-04-18 14:39:01 +02:00
yhampe	9bb0468b2b	RED-8402: Header and footer are not indexed / searched added unit test and file	2024-04-18 14:36:25 +02:00
Kilian Schüttler	c4d9c5df02	Merge branch 'RED-8747-fp' into 'main' RED-8747 - Entities not merged properly - fp See merge request fforesight/layout-parser!131	2024-04-09 16:30:02 +02:00
Corina Olariu	976f408237	RED-8747 - Entities not merged properly - fp - rework the extraction of rulings from the table cells	2024-04-09 14:38:48 +03:00
Corina Olariu	319268c53d	RED-8747 - Entities not merged properly - fp - update test	2024-04-09 12:24:19 +03:00
Corina Olariu	014eba9fc3	RED-8747 - Entities not merged properly - fp - fix typo - add validate table test	2024-04-09 12:14:57 +03:00
Yannik Hampe	9bd8419770	Merge branch 'RED-8402' into 'main' RED-8402: Header and footer are not indexed / searched See merge request fforesight/layout-parser!128	2024-04-08 12:28:06 +02:00
yhampe	c13ff7fbf6	RED-8402: Header and footer are not indexed / searched checkstyle added review comments	2024-04-08 12:17:49 +02:00
yhampe	5d3826e9b9	Merge remote-tracking branch 'origin/RED-8402' into RED-8402	2024-04-08 12:02:47 +02:00
yhampe	0c3194276a	RED-8402: Header and footer are not indexed / searched added headers and footers to simplifiedtext	2024-04-08 12:02:36 +02:00
yhampe	e302d9784e	RED-8402: Header and footer are not indexed / searched added headers and footers to simplifiedtext	2024-04-08 11:59:35 +02:00
Corina Olariu	f185b13f2b	RED-8747 - Entities not merged properly - fp - use the rullings from the found tables instead of all rullings as splitting rullings in the blockification service	2024-04-08 09:42:32 +03:00
Dominique Eifländer	990c376ce6	Merge branch 'RED-8873' into 'main' RED-8773 - Fix images not appearing on specific file See merge request fforesight/layout-parser!123	2024-04-05 10:11:23 +02:00
Kilian Schüttler	bf6a0d770b	Merge branch 'RED-8799' into 'main' RED-8799: LayoutGrid is wrong draw for some tables See merge request fforesight/layout-parser!126	2024-04-04 15:23:12 +02:00
Kilian Schuettler	f18bda1d4e	RED-8799: LayoutGrid is wrong draw for some tables	2024-04-04 13:33:22 +02:00
Maverick Studer	0a11992361	Merge branch 'RED-8702' into 'main' RED-8702: Explore document databases to store entityLog See merge request fforesight/layout-parser!125	2024-04-03 10:00:38 +02:00
Andrei Isvoran	456b8fe4a1	RED-8773 - Fix images not appearing on specific file	2024-04-03 10:20:46 +03:00
maverickstuder	9778ece992	RED-8702: Explore document databases to store entityLog * fix for duplicate images in document structure that are linked to multiple sections	2024-04-02 14:19:14 +02:00
Timo Bejan	8bd0de6263	Merge branch 'RED-8827' into 'main' Red 8827 See merge request fforesight/layout-parser!122	2024-03-22 12:25:36 +01:00
Timo Bejan	5c1708f97f	Issue with merging text blocks multiple times	2024-03-22 12:47:05 +02:00
Timo Bejan	a35d77be2e	ignore mc files	2024-03-22 10:22:00 +02:00
Dominique Eifländer	631160eb22	Merge branch 'RED-8627' into 'main' RED-8627: Fixed scrambled text after sorting See merge request fforesight/layout-parser!120	2024-03-19 11:09:48 +01:00
Dominique Eifländer	8e7e588d26	RED-8627: Fixed scrambled text after sorting	2024-03-19 10:58:36 +01:00
Dominique Eifländer	ac850c2626	Merge branch 'RED-7141' into 'main' RED-7141: Fixed more overlap problems See merge request fforesight/layout-parser!119	2024-03-14 16:46:10 +01:00
Dominique Eifländer	1d765a6baa	RED-7141: Fixed more overlap problems	2024-03-14 16:30:52 +01:00
Dominique Eifländer	c55984aa67	Merge branch 'RED-7141' into 'main' RED-7141: Fixed overlapping blocks See merge request fforesight/layout-parser!118	2024-03-14 09:09:52 +01:00
Dominique Eifländer	27aa418029	RED-7141: Fixed overlapping blocks	2024-03-13 16:14:55 +01:00
Dominique Eifländer	c4edff4696	Merge branch 'RED-7141' into 'main' RED-7141: Readded lost mergeLinesInZones See merge request fforesight/layout-parser!116	2024-03-12 13:49:09 +01:00
Dominique Eifländer	92fd1a72de	RED-7141: Readded lost mergeLinesInZones	2024-03-12 13:42:40 +01:00
Dominique Eifländer	0d3d25e7d7	Merge branch 'RED-7141-hotfix' into 'main' RED-7141: Align backend text sorting with Webviewer sorting See merge request fforesight/layout-parser!115	2024-03-12 11:15:41 +01:00
maverickstuder	956fbff872	RED-7141: Align backend text sorting with Webviewer sorting * hotfix for tables not being detected due to wrong x-y-sorting	2024-03-12 11:06:53 +01:00
Maverick Studer	2488009af1	Merge branch 'RED-8715' into 'main' RED-8715: Improve NearestNeighbor Algorithm in LayoutParser See merge request fforesight/layout-parser!114	2024-03-11 15:10:41 +01:00
maverickstuder	16be2467fd	RED-8715: Improve NearestNeighbor Algorithm in LayoutParser * replaced the old algorithm with an algorithm based on a kd-tree	2024-03-11 14:42:28 +01:00
Timo Bejan	f4cae8a7dc	Merge branch 'Clarifynd' into 'main' Clarifynd See merge request fforesight/layout-parser!113	2024-03-11 10:37:05 +01:00
Timo Bejan	dfc23955d7	Linespacing claryfind	2024-03-11 11:30:51 +02:00
Dominique Eifländer	d6e3d6fe22	Clarifynd	2024-03-11 11:24:58 +02:00
Timo Bejan	bef23e38b5	Merge branch 'clari-30' into 'main' CLARI-30 - forward analysis headers See merge request fforesight/layout-parser!112	2024-03-08 15:51:53 +01:00
Timo Bejan	65ab7a1912	CLARI-30 - forward analysis headers	2024-03-08 16:47:27 +02:00
Timo Bejan	d80231e4a9	Merge branch 'clari-30' into 'main' CLARI-30 - identifier fix for clarifynd See merge request fforesight/layout-parser!111	2024-03-08 15:28:33 +01:00
Timo Bejan	56c07a4491	CLARI-30 - identifier fix for clarifynd	2024-03-08 16:23:27 +02:00
Dominique Eifländer	0b4ad29dcb	Merge branch 'RED-7141' into 'main' RED-7141: Implemented docstrum layout parsing See merge request fforesight/layout-parser!108	2024-03-08 14:27:59 +01:00
Dominique Eifländer	0ad0cd45d6	RED-7141: Moved docstrum to root level of processor package	2024-03-08 14:20:28 +01:00
Dominique Eifländer	d659fe7234	RED-7141: Performance improvments	2024-03-08 10:00:52 +01:00
Dominique Eifländer	cb9127b4f3	RED-7141: Fixed pr finding and improved speed	2024-03-07 16:51:48 +01:00
Timo Bejan	05523585c0	orchestrator/persistence service should control queues	2024-03-06 16:55:44 +02:00
Timo Bejan	4ced572949	orchestrator/persistence service should control queues	2024-03-06 16:53:10 +02:00
Dominique Eifländer	79239b751d	RED-7141: Implemented docstrum layout parsing	2024-03-06 11:18:40 +01:00