Merge branch 'RM-231-bp-bp' into 'release/0.159.x'

RM-231: missing whitespace in name See merge request fforesight/layout-parser!265
RM-231: missing whitespace in name
2025-01-14 13:05:03 +01:00 · 2025-01-14 13:00:00 +01:00 · 2025-01-10 12:33:14 +01:00 · 2025-01-10 12:10:35 +01:00 · 2024-12-06 14:56:05 +01:00 · 2024-12-06 14:49:35 +01:00
144 changed files with 4744 additions and 1830 deletions
@@ -42,6 +42,19 @@ tasks.jacocoTestReport {
 }

 allprojects {
+
+    tasks.withType<Javadoc> {
+        options {
+            this as StandardJavadocDocletOptions
+            addBooleanOption("Xdoclint:none", true)
+            addStringOption("Xmaxwarns", "1")
+        }
+    }
+
+    pmd {
+        setConsoleOutput(true)
+    }
+
    publishing {
        publications {
            create<MavenPublication>(name) {
@@ -64,6 +77,7 @@ java {
    withJavadocJar()
 }

+
 repositories {
    mavenLocal()
    mavenCentral()
@@ -19,6 +19,7 @@ public class SimplifiedText {
    @Schema(description = "Number of pages in the entire document.")
    private int numberOfPages;
    @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
+    @Builder.Default
    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();

 }
@@ -25,7 +25,13 @@ dependencies {
    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
    implementation("org.jgrapht:jgrapht-core:1.5.2")
+    implementation("org.apache.pdfbox:jbig2-imageio:3.0.4")
+    implementation("com.github.jai-imageio:jai-imageio-core:1.4.0")
+    implementation("com.github.jai-imageio:jai-imageio-jpeg2000:1.4.0")
    implementation("org.tinspin:tinspin-indexes:2.1.3")
    implementation("org.commonmark:commonmark:0.22.0")
    implementation("org.commonmark:commonmark-ext-gfm-tables:0.22.0")
+    implementation("com.pdftron:PDFNet:10.11.0")
+    implementation("org.apache.commons:commons-text:1.12.0")
+
 }
@@ -2,13 +2,15 @@ package com.knecon.fforesight.service.layoutparser.processor;

 import static java.lang.String.format;

-import java.awt.geom.Point2D;
+import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
@@ -24,7 +26,10 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.No
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.markdown.MarkdownMapper;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.MarkdownMapper;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -39,7 +44,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableO
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
@@ -47,7 +52,6 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
@@ -57,9 +61,6 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClarifyndClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.Box;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.GraphicExtractorService;
@@ -69,6 +70,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDF
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import io.micrometer.observation.Observation;
 import io.micrometer.observation.ObservationRegistry;
@@ -90,10 +92,7 @@ public class LayoutParsingPipeline {
    CvTableParsingAdapter cvTableParsingAdapter;
    LayoutParsingStorageService layoutParsingStorageService;
    SectionsBuilderService sectionsBuilderService;
-    RedactManagerClassificationService redactManagerClassificationService;
-    DocuMineClassificationService docuMineClassificationService;
    SimplifiedSectionTextService simplifiedSectionTextService;
-    BodyTextFrameService bodyTextFrameService;
    RulingCleaningService rulingCleaningService;
    TableExtractionService tableExtractionService;
    DocuMineBlockificationService docuMineBlockificationService;
@@ -103,12 +102,12 @@ public class LayoutParsingPipeline {
    LayoutGridService layoutGridService;
    ObservationRegistry observationRegistry;
    VisualLayoutParsingAdapter visualLayoutParsingAdapter;
-    ClarifyndClassificationService clarifyndClassificationService;
    GraphicExtractorService graphicExtractorService;
    OutlineExtractorService outlineExtractorService;
    OutlineValidationService outlineValidationService;
    TOCEnrichmentService tocEnrichmentService;
    LayoutparserSettings settings;
+    ClassificationService classificationService;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -137,17 +136,16 @@ public class LayoutParsingPipeline {
        log.info("Building document graph for {}", layoutParsingRequest.identifier());

        Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null //
-                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
-                                                           classificationDocument);
+                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument);

        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());

-        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, layoutParsingRequest.visualLayoutParsingFileId().isPresent());
+        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false);

        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());

        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-        if(layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
+        if (layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
            layoutParsingStorageService.storeMarkdownFile(layoutParsingRequest.documentMarkdownFileStorageId().get(), new MarkdownMapper().toMarkdownContent(documentGraph));
        }
        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
@@ -245,7 +243,7 @@ public class LayoutParsingPipeline {
        OutlineObject lastProcessedOutlineObject = null;

        // parsing the structure elements could be useful as well
-        if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD && layoutParsingType != LayoutParsingType.DOCUMINE_OLD) {
+        if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD) {
            classificationDocument.setOutlineObjectTree(outlineExtractorService.getOutlineObjectTree(originDocument));
        }

@@ -271,11 +269,16 @@ public class LayoutParsingPipeline {
            stripper.setStartPage(pageNumber);
            stripper.setEndPage(pageNumber);
            stripper.setPdpage(pdPage);
-            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
-                stripper.setSortByPosition(true);
-            }
            stripper.getText(originDocument);
-            List<TextPositionSequence> words = stripper.getTextPositionSequences();
+            List<Word> words = stripper.getWords();
+
+//            rotateDirAdjExactly(words, pdPage); // works really well for many highly rotated documents (e.g. VV-331340.pdf), but it decreases the headline performance by 1.3%, so I am leaving it out for now
+
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
+                var lines = TextPositionOperations.groupByLine(new HashSet<>(words));
+                classificationDocument.getLayoutDebugLayer().addLineVisualizationsFromNestedTextPosition(lines, pageNumber);
+                words = TextPositionOperations.sortLines(lines);
+            }
            classificationDocument.getLayoutDebugLayer().addTextVisualizations(words, pageNumber);

            PDRectangle pdr = pdPage.getMediaBox();
@@ -293,7 +296,7 @@ public class LayoutParsingPipeline {

            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(words, cleanRulings);

-            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getTextPositionSequences(), false);
+            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getWords(), false);

            pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>())
                    .addAll(graphics.stream()
@@ -306,7 +309,7 @@ public class LayoutParsingPipeline {

            ClassificationPage classificationPage = switch (layoutParsingType) {
                case REDACT_MANAGER_OLD ->
-                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings, classificationDocument.getLayoutDebugLayer());
+                        redactManagerBlockificationService.blockify(stripper.getWords(), cleanRulings, classificationDocument.getLayoutDebugLayer());
                case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
                case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
                        docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
@@ -321,18 +324,19 @@ public class LayoutParsingPipeline {
            classificationPage.setPageWidth(cropbox.getWidth());
            classificationPage.setPageHeight(cropbox.getHeight());

-            if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD && layoutParsingType != LayoutParsingType.DOCUMINE_OLD) {
-                List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber - 1, new ArrayList<>());
+            if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD) {
+                List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber, new ArrayList<>());

                OutlineObject notFoundOutlineObject = null;
                if (lastProcessedOutlineObject != null && !lastProcessedOutlineObject.isFound()) {
-                    lastProcessedOutlineObject.setPoint(new Point2D.Float(0, cropbox.getHeight()));
+                    lastProcessedOutlineObject.resetPoint();
                    notFoundOutlineObject = lastProcessedOutlineObject;
                }
                if (!outlineObjects.isEmpty()) {
                    classificationPage.setOutlineObjects(outlineObjects);
                    lastProcessedOutlineObject = blockificationPostprocessingService.sanitizeOutlineBlocks(classificationPage, notFoundOutlineObject);
                }
+                classificationDocument.getLayoutDebugLayer().addOutlineObjects(outlineObjects, pageInformation);
            }

            classificationDocument.getLayoutDebugLayer().addMarkedContentVisualizations(stripper.getMarkedContents(), pageNumber);
@@ -363,27 +367,9 @@ public class LayoutParsingPipeline {

        originDocument.close();

-        log.info("Calculating BodyTextFrame for {}", identifier);
-        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
-        for (ClassificationPage page : classificationDocument.getPages()) {
-            classificationDocument.getLayoutDebugLayer().addCleanRulingVisualization(page.getCleanRulings(), page.getPageNumber());
-        }
-        log.info("Classify TextBlocks for {}", identifier);
-        switch (layoutParsingType) {
-            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
-                    redactManagerClassificationService.classifyDocument(classificationDocument);
-            case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
-        }
+        classificationService.classify(classificationDocument, layoutParsingType, identifier);

-        List<TextPageBlock> headlines = classificationDocument.getPages()
-                .stream()
-                .flatMap(classificationPage -> classificationPage.getTextBlocks()
-                        .stream()
-                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
-                        .map(tb -> (TextPageBlock) tb))
-                .toList();
-        TableOfContents tableOfContents = outlineValidationService.createToC(headlines);
+        TableOfContents tableOfContents = outlineValidationService.createToC(classificationDocument);
        classificationDocument.setTableOfContents(tableOfContents);

        log.info("Building Sections for {}", identifier);
@@ -397,6 +383,32 @@ public class LayoutParsingPipeline {
    }


+    private static void rotateDirAdjExactly(List<Word> words, PDPage pdPage) {
+
+        for (TextDirection dir : TextDirection.values()) {
+
+            double averageRotation = words.stream()
+                    .map(Word::getTextPositions)
+                    .flatMap(Collection::stream)
+                    .filter(pos -> pos.getDir().equals(dir))
+                    .mapToDouble(RedTextPosition::getExactDir).average().orElse(0);
+
+            if (averageRotation == 0) {
+                continue;
+            }
+
+            AffineTransform rotateInstance = AffineTransform.getRotateInstance(averageRotation, pdPage.getMediaBox().getWidth() / 2, pdPage.getMediaBox().getHeight() / 2);
+
+            for (Word word : words) {
+                if (!dir.equals(word.getDir())) {
+                    continue;
+                }
+                word.transform(rotateInstance);
+            }
+        }
+    }
+
+
    private void addNumberOfPagesToTrace(int numberOfPages, long size) {

        if (observationRegistry.getCurrentObservation() != null) {
@@ -438,10 +450,10 @@ public class LayoutParsingPipeline {
        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
            if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getSequences() == null) {
+                if (((TextPageBlock) textBlock).getWords() == null) {
                    continue;
                }
-                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                for (Word word : ((TextPageBlock) textBlock).getWords()) {
                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
                    classificationPage.getFontCounter().add(word.getFont());
                    classificationPage.getFontSizeCounter().add(word.getFontSize());
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum;

 import java.util.ArrayList;
+import java.util.EnumMap;
 import java.util.List;
 import java.util.stream.Collectors;

@@ -17,7 +18,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.Zon
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;

 import lombok.RequiredArgsConstructor;
@@ -26,6 +27,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
 public class DocstrumSegmentationService {

+    public static final double SAME_DIRECTION_THRESHOLD = 0.9;
    private final NearestNeighbourService nearestNeighbourService;
    private final SpacingService spacingService;
    private final LineBuilderService lineBuilderService;
@@ -33,23 +35,54 @@ public class DocstrumSegmentationService {
    private final ReadingOrderService readingOrderService;


-    public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
+    public List<Zone> segmentPage(List<Word> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {

-        List<Zone> zones = new ArrayList<>();
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.ZERO));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.QUARTER_CIRCLE));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.HALF_CIRCLE));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.THREE_QUARTER_CIRCLE));
+        EnumMap<TextDirection, Integer> directionCounts = new EnumMap<>(TextDirection.class);

-        return readingOrderService.resolve(zones, xyOrder);
+        List<Zone> newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.ZERO);
+        directionCounts.put(TextDirection.ZERO, newZones.size());
+        List<Zone> zones = new ArrayList<>(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.QUARTER_CIRCLE);
+        directionCounts.put(TextDirection.QUARTER_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.HALF_CIRCLE);
+        directionCounts.put(TextDirection.HALF_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.THREE_QUARTER_CIRCLE);
+        directionCounts.put(TextDirection.THREE_QUARTER_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        return readingOrderService.resolve(zones, xyOrder, mostSameDirection(directionCounts));
    }


-    private List<Zone> computeZones(List<TextPositionSequence> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
+    private boolean mostSameDirection(EnumMap<TextDirection, Integer> directionCounts) {
+
+        int total = directionCounts.values()
+                .stream()
+                .mapToInt(i -> i).sum();
+
+        if ((double) directionCounts.get(TextDirection.ZERO) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.QUARTER_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.HALF_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.THREE_QUARTER_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        }
+        return false;
+    }
+
+
+    private List<Zone> computeZones(List<Word> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {

        List<RedTextPosition> positions = textPositions.stream()
                .filter(t -> t.getDir() == direction)
-                .map(TextPositionSequence::getTextPositions)
+                .map(Word::getTextPositions)
                .flatMap(List::stream)
                .toList();

@@ -15,10 +15,16 @@ public class AngleFilter {

    public boolean matches(Neighbor neighbor) {

+        return matches(neighbor.getAngle());
+    }
+
+
+    public boolean matches(double angle) {
+
        if (lowerAngle <= upperAngle) {
-            return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
+            return lowerAngle <= angle && angle < upperAngle;
        } else {
-            return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
+            return lowerAngle <= angle || angle < upperAngle;
        }
    }

@@ -7,8 +7,12 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;

@Data
+@SuperBuilder
+@NoArgsConstructor
 public abstract class BoundingBox {

    // Java coordinate system: (0, 0) is always upper left, x is increasing left to right and y is increasing from top to bottom.
@@ -19,7 +23,7 @@ public abstract class BoundingBox {
    // This rotates completely in 90 degree steps with page rotation.
    // Needs to be used when writing to a PDF.
    // Also, these are definitely correct and should be used whenever possible.
-    protected Rectangle2D bBoxInitialUserSpace;
+    protected Rectangle2D bBoxPdf;

    protected static final float VERTICAL_COMPARISON_THRESHOLD = 0.4f;

@@ -50,25 +54,25 @@ public abstract class BoundingBox {

    public double getPdfMinX() {

-        return bBoxInitialUserSpace.getMinX();
+        return bBoxPdf.getMinX();
    }


    public double getPdfMaxX() {

-        return bBoxInitialUserSpace.getMaxX();
+        return bBoxPdf.getMaxX();
    }


    public double getPdfMinY() {

-        return bBoxInitialUserSpace.getMinY();
+        return bBoxPdf.getMinY();
    }


    public double getPdfMaxY() {

-        return bBoxInitialUserSpace.getMaxY();
+        return bBoxPdf.getMaxY();
    }


@@ -129,13 +133,31 @@ public abstract class BoundingBox {
    }


-    public boolean intersectsY(BoundingBox other) {
+    public boolean intersectsX(BoundingBox other, float threshold) {
+
+        return this.getX() - threshold <= other.getMaxX() && this.getMaxX() + threshold >= other.getX();
+    }
+
+
+    public boolean intersectsPdf(BoundingBox other) {
+
+        return this.intersectsXPdf(other) && this.intersectsYPdf(other);
+    }
+
+
+    public boolean intersectsPdf(BoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsXPdf(other, xThreshold) && this.intersectsYPdf(other, yThreshold);
+    }
+
+
+    public boolean intersectsYPdf(BoundingBox other) {

        return this.getPdfMinY() <= other.getPdfMaxY() && this.getPdfMaxY() >= other.getPdfMinY();
    }


-    public boolean intersectsYJava(BoundingBox other) {
+    public boolean intersectsY(BoundingBox other) {

        return this.getY() <= other.getMaxY() && this.getMaxY() >= other.getY();
    }
@@ -143,25 +165,31 @@ public abstract class BoundingBox {

    public boolean intersectsY(BoundingBox other, float threshold) {

+        return this.getY() - threshold <= other.getMaxY() && this.getMaxY() + threshold >= other.getY();
+    }
+
+
+    public boolean intersectsYPdf(BoundingBox other, float threshold) {
+
        return this.getPdfMinY() - threshold <= other.getPdfMaxY() && this.getPdfMaxY() + threshold >= other.getPdfMinY();
    }


-    public boolean intersectsX(BoundingBox other) {
+    public boolean intersectsXPdf(BoundingBox other) {

        return this.getPdfMinX() <= other.getPdfMaxX() && this.getPdfMaxX() >= other.getPdfMinX();
    }


-    public boolean intersectsXJava(BoundingBox other) {
+    public boolean intersectsX(BoundingBox other) {

        return this.getX() <= other.getMaxX() && this.getMaxX() >= other.getMinX();
    }


-    public boolean intersectsX(BoundingBox other, float threshold) {
+    public boolean intersectsXPdf(BoundingBox other, float threshold) {

-        return this.getPdfMinX() - threshold <= other.getMaxX() && this.getMaxX() + threshold >= other.getPdfMinX();
+        return this.getPdfMinX() - threshold <= other.getPdfMaxX() && this.getMaxX() + threshold >= other.getPdfMinX();
    }


@@ -170,8 +198,8 @@ public abstract class BoundingBox {
        this.bBox = components.stream()
                .map(BoundingBox::getBBox)
                .collect(RectangleTransformations.collectBBox());
-        this.bBoxInitialUserSpace = components.stream()
-                .map(BoundingBox::getBBoxInitialUserSpace)
+        this.bBoxPdf = components.stream()
+                .map(BoundingBox::getBBoxPdf)
                .collect(RectangleTransformations.collectBBox());
    }

@@ -197,57 +225,55 @@ public abstract class BoundingBox {

    public double horizontalDistance(BoundingBox other) {

-        Rectangle2D left;
-        Rectangle2D right;
-        if (this.leftOf(other)) {
-            left = this.getBBox();
-            right = other.getBBox();
-        } else {
-            left = other.getBBox();
-            right = this.getBBox();
-        }
+        double rect1Right = getMaxX();
+        double rect1Left = getMinX();
+        double rect2Right = other.getMaxX();
+        double rect2Left = other.getMinX();

-        return Math.max(0, right.getMinX() - left.getMaxX());
+        if (rect1Left > rect2Right || rect2Left > rect1Right) {
+            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        } else {
+            return 0;
+        }
    }


    public double verticalDistance(BoundingBox other) {

-        Rectangle2D bottom;
-        Rectangle2D top;
-        if (this.isAbove(other)) {
-            top = this.getBBox();
-            bottom = other.getBBox();
-        } else {
-            bottom = this.getBBox();
-            top = other.getBBox();
-        }
+        double rect1Top = getMaxY();
+        double rect1Bottom = getMinY();
+        double rect2Top = other.getMaxY();
+        double rect2Bottom = other.getMinY();

-        return Math.max(0, bottom.getMinY() - top.getMaxY());
+        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
+            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        } else {
+            return 0;
+        }
    }


    public boolean rightOf(BoundingBox other) {

-        return this.intersectsYJava(other) && other.getMaxX() <= this.getMinX();
+        return this.intersectsY(other) && other.getMaxX() <= this.getMinX();
    }


    public boolean leftOf(BoundingBox other) {

-        return this.intersectsYJava(other) && other.getMinX() >= this.getMaxX();
+        return this.intersectsY(other) && other.getMinX() >= this.getMaxX();
    }


    public boolean isAbove(BoundingBox other) {

-        return this.intersectsXJava(other) && other.getMinY() >= this.getMaxY();
+        return this.intersectsX(other) && other.getMinY() >= this.getMaxY();
    }


    public boolean isBelow(BoundingBox other) {

-        return this.intersectsXJava(other) && this.getMinY() >= other.getMaxY();
+        return this.intersectsX(other) && this.getMinY() >= other.getMaxY();
    }

 }
@@ -35,7 +35,7 @@ public class Character {

    public double getHeight() {

-        return textPosition.getHeightDir();
+        return textPosition.getHeightDirAdj();
    }


@@ -65,9 +65,9 @@ public class Character {
        double s = Math.sin(-0);
        double c = Math.cos(-0);
        xs[0] = c * x - s * y;
-        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
+        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDirAdj());
        xs[2] = c * other.x - s * other.y;
-        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
+        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDirAdj());
        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
        Arrays.sort(xs);
        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
@@ -1,20 +1,30 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD_ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.STANDARD;
+
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;

 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;

 import lombok.Data;
 import lombok.EqualsAndHashCode;

@Data
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
-public class Line extends BoundingBox {
+@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
+public class Line extends TextBoundingBox {

-    private static final double WORD_DISTANCE_MULTIPLIER = 0.18;
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.17;

    @EqualsAndHashCode.Include
    private final double x0;
@@ -28,8 +38,10 @@ public class Line extends BoundingBox {

    private final double height;

+    private FontStyle fontStyle;
+
    private final List<Character> characters;
-    private final List<TextPositionSequence> words = new ArrayList<>();
+    private final List<Word> words = new ArrayList<>();


    public Line(List<Character> characters, double wordSpacing) {
@@ -67,6 +79,29 @@ public class Line extends BoundingBox {
        height = computeHeight();
        computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
        buildBBox();
+        computeFontStyle();
+    }
+
+
+    private void computeFontStyle() {
+
+        EnumMap<FontStyle, AtomicInteger> fontStyleCounter = new EnumMap<>(FontStyle.class);
+        for (FontStyle fontStyle : FontStyle.values()) {
+            fontStyleCounter.put(fontStyle, new AtomicInteger(0));
+        }
+        for (Word word : words) {
+            switch (word.getFontStyle()) {
+                case STANDARD -> fontStyleCounter.get(FontStyle.REGULAR).getAndIncrement();
+                case BOLD -> fontStyleCounter.get(FontStyle.BOLD).getAndIncrement();
+                case ITALIC -> fontStyleCounter.get(FontStyle.ITALIC).getAndIncrement();
+                case BOLD_ITALIC -> fontStyleCounter.get(FontStyle.BOLD_ITALIC).getAndIncrement();
+            }
+        }
+        fontStyle = fontStyleCounter.entrySet()
+                .stream()
+                .max(Comparator.comparing(entry -> entry.getValue().get()))
+                .map(Map.Entry::getKey)
+                .orElse(FontStyle.REGULAR);
    }


@@ -124,14 +159,17 @@ public class Line extends BoundingBox {

    private void computeWords(double wordSpacing) {

-        TextPositionSequence word = new TextPositionSequence();
+        // Imo, the width of space should be scaled with the font size, but it only depends on the median distance between horizontal neighbours.
+        // If there are large differences in fontsize on a page, this might lead to missing spaces for the smaller fonts and too many for larger fonts.
+        // I've just now changed the scaling factor. If you come across this comment with missing whitespaces again, try scaling the fontsize instead of simply changing the factor again.
+        Word word = new Word();
        Character previous = null;
        for (Character current : characters) {
            if (previous != null) {
                double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
                if (dist > wordSpacing) {
                    words.add(word);
-                    word = new TextPositionSequence();
+                    word = new Word();
                }
            }
            word.getTextPositions().add(current.getTextPosition());
@@ -144,8 +182,8 @@ public class Line extends BoundingBox {
    private void buildBBox() {

        this.setToBBoxOfComponents(characters.stream()
-                                       .map(Character::getTextPosition)
-                                       .toList());
+                                           .map(Character::getTextPosition)
+                                           .toList());
    }


@@ -0,0 +1,180 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.SuperBuilder;
+
+@Getter
+@Setter
+@SuperBuilder
+@NoArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+public abstract class TextBoundingBox extends BoundingBox {
+
+    protected Rectangle2D bBoxDirAdj;
+
+    protected TextDirection dir;
+
+
+    @Override
+    public void setToBBoxOfComponents(List<? extends BoundingBox> components) {
+
+        super.setToBBoxOfComponents(components);
+        this.bBoxDirAdj = components.stream()
+                .filter(c -> c instanceof TextBoundingBox)
+                .map(c -> (TextBoundingBox) c)
+                .map(TextBoundingBox::getBBoxDirAdj)
+                .collect(RectangleTransformations.collectBBox());
+
+        Set<TextDirection> textDirections = components.stream()
+                .filter(c -> c instanceof TextBoundingBox)
+                .map(c -> (TextBoundingBox) c)
+                .map(TextBoundingBox::getDir)
+                .collect(Collectors.toSet());
+
+        if (textDirections.isEmpty()) {
+            dir = TextDirection.ZERO;
+        } else if (textDirections.size() > 1) {
+            throw new IllegalArgumentException("More than one text direction found");
+        } else {
+            dir = textDirections.iterator().next();
+        }
+    }
+
+
+    public double getXDirAdj() {
+
+        return this.bBoxDirAdj.getX();
+    }
+
+
+    public double getYDirAdj() {
+
+        return this.bBoxDirAdj.getY();
+    }
+
+
+    public double getWidthDirAdj() {
+
+        return this.bBoxDirAdj.getWidth();
+    }
+
+
+    public double getHeightDirAdj() {
+
+        return this.bBoxDirAdj.getHeight();
+    }
+
+
+    public double getMaxXDirAdj() {
+
+        return this.bBoxDirAdj.getMaxX();
+    }
+
+
+    public double getMaxYDirAdj() {
+
+        return this.bBoxDirAdj.getMaxY();
+    }
+
+
+    public double getCenterYDirAdj() {
+
+        return this.bBoxDirAdj.getCenterY();
+    }
+
+
+    public double getCenterXDirAdj() {
+
+        return this.bBoxDirAdj.getCenterX();
+    }
+
+
+    public double horizontalDistanceDirAdj(TextBoundingBox other) {
+
+        double rect1Right = getMaxXDirAdj();
+        double rect1Left = getXDirAdj();
+        double rect2Right = other.getMaxXDirAdj();
+        double rect2Left = other.getXDirAdj();
+
+        if (rect1Left > rect2Right || rect2Left > rect1Right) {
+            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public double verticalDistanceDirAdj(TextBoundingBox other) {
+
+        double rect1Top = getMaxYDirAdj();
+        double rect1Bottom = getYDirAdj();
+        double rect2Top = other.getMaxYDirAdj();
+        double rect2Bottom = other.getYDirAdj();
+
+        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
+            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public boolean intersectsDirAdj(TextBoundingBox other) {
+
+        return this.intersectsXDirAdj(other) && this.intersectsYDirAdj(other);
+    }
+
+
+    public boolean intersectsDirAdj(TextBoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsXDirAdj(other, xThreshold) && this.intersectsYDirAdj(other, yThreshold);
+    }
+
+
+    public boolean intersectsXDirAdj(TextBoundingBox other, float threshold) {
+
+        return this.getXDirAdj() - threshold <= other.getMaxXDirAdj() && this.getMaxXDirAdj() + threshold >= other.getXDirAdj();
+    }
+
+
+    public boolean intersectsXDirAdj(TextBoundingBox other) {
+
+        return this.getXDirAdj() <= other.getMaxXDirAdj() && this.getMaxXDirAdj() >= other.getXDirAdj();
+    }
+
+
+    public boolean intersectsYDirAdj(TextBoundingBox other) {
+
+        return this.getYDirAdj() <= other.getMaxYDirAdj() && this.getMaxYDirAdj() >= other.getYDirAdj();
+    }
+
+
+    public boolean intersectsYDirAdj(TextBoundingBox other, float threshold) {
+
+        return this.getYDirAdj() - threshold <= other.getMaxYDirAdj() && this.getMaxYDirAdj() + threshold >= other.getYDirAdj();
+    }
+
+
+    public boolean isAboveDirAdj(TextBoundingBox other) {
+
+        return other.isBelow(this);
+    }
+
+
+    public boolean isBelowDirAdj(TextBoundingBox other) {
+
+        return this.intersectsXDirAdj(other) && this.getYDirAdj() >= other.getMaxYDirAdj();
+    }
+
+}
@@ -28,4 +28,10 @@ public class UnionFind<T> extends org.jgrapht.alg.util.UnionFind<T> {
        return setRep.values();
    }

+
+    public Collection<T> getElements() {
+
+        return getParentMap().keySet();
+    }
+
 }
@@ -6,9 +6,11 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.Data;
+import lombok.EqualsAndHashCode;

@Data
-public class Zone extends BoundingBox {
+@EqualsAndHashCode(callSuper = false)
+public class Zone extends TextBoundingBox {

    private List<Line> lines;

@@ -1,15 +1,17 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;

+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
+import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;

@@ -19,21 +21,30 @@ public class ReadingOrderService {
    private static final double THRESHOLD = 5;
    public static final double MULTI_COLUMN_DETECTION_THRESHOLD = 1.5;

+    private static final Comparator<TextBoundingBox> COMPARATOR = //
+            Comparator.comparing(TextBoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(TextBoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD));

-    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder) {
+    private static final Comparator<TextBoundingBox> COMPARATOR_DIR_ADJ = //
+            Comparator.comparing(TextBoundingBox::getYDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD));
+
+
+    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder, boolean useDirAdjCoords) {

        if (zones.isEmpty() || zones.size() == 1) {
            return zones;
        }

        if (xyReadingOrder) {
-            return resolveSingleColumnReadingOrder(zones);
+            return resolveSingleColumnReadingOrder(zones, useDirAdjCoords);
        }

        Map<Long, Integer> histogram = new HashMap<>();
        for (Zone zone : zones) {
-            long minY = Math.round(zone.getBBox().getMinY());
-            long maxY = Math.round(zone.getBBox().getMaxY());
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            long minY = Math.round(bbox.getMinY());
+            long maxY = Math.round(bbox.getMaxY());
            for (long i = minY; i <= maxY; i++) {
                histogram.put(i, histogram.getOrDefault(i, 0) + 1);
            }
@@ -43,24 +54,32 @@ public class ReadingOrderService {
                    .stream()
                    .mapToInt(Integer::intValue).average()
                    .orElse(1) < MULTI_COLUMN_DETECTION_THRESHOLD) {
-            return resolveSingleColumnReadingOrder(zones);
+            return resolveSingleColumnReadingOrder(zones, useDirAdjCoords);
        } else {

-            return resolveMultiColumnReadingOder(zones);
+            return resolveMultiColumnReadingOder(zones, useDirAdjCoords);
        }

    }


-    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones) {
+    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones, boolean useDirAdjCoords) {

-        zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                           .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+        if (useDirAdjCoords) {
+            return zones.stream()
+                    .collect(Collectors.groupingBy(TextBoundingBox::getDir)).values()
+                    .stream()
+                    .flatMap(words -> words.stream()
+                            .sorted(COMPARATOR_DIR_ADJ))
+                    .toList();
+        }
+
+        zones.sort(COMPARATOR);
        return zones;
    }


-    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
+    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones, boolean useDirAdjCoords) {

        // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
        // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
@@ -69,11 +88,12 @@ public class ReadingOrderService {
        double maxX = Double.NEGATIVE_INFINITY;

        for (Zone zone : zones) {
-            if (zone.getX() < minX) {
-                minX = zone.getX();
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            if (bbox.getX() < minX) {
+                minX = zone.getXDirAdj();
            }
-            if (zone.getX() + zone.getWidth() > maxX) {
-                maxX = zone.getX() + zone.getWidth();
+            if (bbox.getMaxX() > maxX) {
+                maxX = zone.getMaxXDirAdj();
            }
        }

@@ -82,24 +102,27 @@ public class ReadingOrderService {
        List<Zone> leftOf = new ArrayList<>();
        List<Zone> rightOf = new ArrayList<>();
        List<Zone> middle = new ArrayList<>();
+
        for (Zone zone : zones) {
-            if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            if (bbox.getX() < midLineXCoordinate && bbox.getX() + bbox.getWidth() < midLineXCoordinate) {
                leftOf.add(zone);
-            } else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
+            } else if (bbox.getX() > midLineXCoordinate && bbox.getX() + bbox.getWidth() > midLineXCoordinate) {
                rightOf.add(zone);
            } else {
                middle.add(zone);
            }
        }

-        leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                            .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                             .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                            .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+        if (useDirAdjCoords) {
+            leftOf.sort(COMPARATOR_DIR_ADJ);
+            rightOf.sort(COMPARATOR_DIR_ADJ);
+            middle.sort(COMPARATOR_DIR_ADJ);
+        } else {
+            leftOf.sort(COMPARATOR);
+            rightOf.sort(COMPARATOR);
+            middle.sort(COMPARATOR);
+        }
 /*
        List<Zone> leftNotIntersecting = new ArrayList<>();
        for (Zone leftZone : leftOf) {
@@ -151,8 +174,9 @@ public class ReadingOrderService {

        while (itty.hasNext()) {
            Zone current = itty.next();
+            Rectangle2D bbox = useDirAdjCoords ? current.getBBoxDirAdj() : current.getBBox();
            for (int i = 0; i < sortedZones.size(); i++) {
-                if (current.getY() < sortedZones.get(i).getY()) {
+                if (bbox.getY() < sortedZones.get(i).getY()) {
                    sortedZones.add(i, current);
                    itty.remove();
                    break;
@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;

+import static com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier.numericalIdentifierPattern;
+
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -21,7 +23,7 @@ public class ZoneBuilderService {
    private static final double MIN_HORIZONTAL_DISTANCE_MULTIPLIER = -0.5;
    private static final double MAX_VERTICAL_DISTANCE_MULTIPLIER = 1.2;

-    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -3.0;
+    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -7;

    private static final double MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER = 0.5;

@@ -38,7 +40,7 @@ public class ZoneBuilderService {

        double minHorizontalDistance = characterSpacing * MIN_HORIZONTAL_DISTANCE_MULTIPLIER;
        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_DISTANCE_MULTIPLIER;
-        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
+        double minHorizontalMergeDistance = lineSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
        double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;

        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
@@ -54,11 +56,26 @@ public class ZoneBuilderService {
                    return;
                }

-                double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
-                scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
+//                if (!innerLine.getFontStyle().equals(outerLine.getFontStyle()) //
+//                    && !outerLine.intersectsY(innerLine, -2f)) {
+//                    return;
+//                }

-                double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
-                double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
+                double horizontalScale = Math.min(outerLine.getHeightDirAdj(), innerLine.getHeightDirAdj()) / meanHeight;
+                horizontalScale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(horizontalScale, MAX_LINE_SIZE_SCALE));
+                double verticalScale = horizontalScale;
+
+//                if (innerLine.toString().endsWith(":")
+//                    || outerLine.toString().endsWith(":")
+//                    || numericalIdentifierPattern.matcher(innerLine.toString()).matches()
+//                    || numericalIdentifierPattern.matcher(outerLine.toString()).matches()) {
+//
+//                    horizontalScale *= 5;
+//                    verticalScale /= 10;
+//                }
+
+                double horizontalDistance = outerLine.horizontalDistance(innerLine) / horizontalScale;
+                double verticalDistance = outerLine.verticalDistance(innerLine) / verticalScale;

                if ((!(minHorizontalDistance <= horizontalDistance) || !(verticalDistance <= maxVerticalDistance)) //
                    && (!(minHorizontalMergeDistance <= horizontalDistance) || !(verticalDistance <= maxVerticalMergeDistance))) {
@@ -87,7 +104,7 @@ public class ZoneBuilderService {
        double weights = 0.0;
        for (Line line : lines) {
            double weight = line.getLength();
-            meanHeight += line.getHeight() * weight;
+            meanHeight += line.getHeightDirAdj() * weight;
            weights += weight;
        }
        meanHeight /= weights;
@@ -7,9 +7,12 @@ public class DoubleUtils {
        if (Double.isNaN(d1) || Double.isNaN(d2)) {
            return Double.compare(d1, d2);
        }
-        long i1 = Math.round(d1 / (precision == 0 ? 1 : precision));
-        long i2 = Math.round(d2 / (precision == 0 ? 1 : precision));
-        return Long.compare(i1, i2);
+
+        if (Math.abs(d1 - d2) < precision) {
+            return 0;
+        }
+
+        return Double.compare(d1, d2);
    }

 }
@@ -18,6 +18,7 @@ import lombok.RequiredArgsConstructor;

@Data
@RequiredArgsConstructor
+
 public class ClassificationPage {

    @NonNull
@@ -25,7 +26,7 @@ public class ClassificationPage {

    private List<OutlineObject> outlineObjects = new ArrayList<>();

-    private List<AbstractPageBlock> headlines  = new ArrayList<>();
+    private List<AbstractPageBlock> headlines = new ArrayList<>();

    private List<ClassifiedImage> images = new ArrayList<>();

@@ -44,7 +45,7 @@ public class ClassificationPage {
    private float pageWidth;
    private float pageHeight;

-    CleanRulings cleanRulings;
+    private CleanRulings cleanRulings;

    private Map<String, List<Rectangle2D>> markedContentBboxPerType = new HashMap<>();

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;

-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -12,10 +11,14 @@ import lombok.Getter;
@Getter
 public class FloatFrequencyCounter {

-    Map<Float, Integer> countPerValue = new HashMap<>();
+    Map<Double, Integer> countPerValue = new HashMap<>();
+    boolean changed;
+    Double mostPopularCache;


-    public void add(float value) {
+    public void add(double value) {
+
+        changed = true;

        if (!countPerValue.containsKey(value)) {
            countPerValue.put(value, 1);
@@ -25,9 +28,11 @@ public class FloatFrequencyCounter {
    }


-    public void addAll(Map<Float, Integer> otherCounter) {
+    public void addAll(Map<Double, Integer> otherCounter) {

-        for (Map.Entry<Float, Integer> entry : otherCounter.entrySet()) {
+        changed = true;
+
+        for (Map.Entry<Double, Integer> entry : otherCounter.entrySet()) {
            if (countPerValue.containsKey(entry.getKey())) {
                countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
            } else {
@@ -37,36 +42,36 @@ public class FloatFrequencyCounter {
    }


-    public Float getMostPopular() {
+    public Double getMostPopular() {

-        Map.Entry<Float, Integer> mostPopular = null;
-        for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
-                mostPopular = entry;
+        if (changed || mostPopularCache == null) {
+            Map.Entry<Double, Integer> mostPopular = null;
+            for (Map.Entry<Double, Integer> entry : countPerValue.entrySet()) {
+                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
+                    mostPopular = entry;
+                }
            }
+            mostPopularCache = mostPopular != null ? mostPopular.getKey() : 0;
+            changed = false;
        }
-        return mostPopular != null ? mostPopular.getKey() : null;
+
+        return mostPopularCache;
    }


-    public List<Float> getHighterThanMostPopular() {
+    public List<Double> getValuesInReverseOrder() {

-        Float mostPopular = getMostPopular();
-        List<Float> higher = new ArrayList<>();
-        for (Float value : countPerValue.keySet()) {
-            if (value > mostPopular) {
-                higher.add(value);
-            }
-        }
-
-        return higher.stream().sorted(Collections.reverseOrder()).collect(Collectors.toList());
+        return countPerValue.keySet()
+                .stream()
+                .sorted(Collections.reverseOrder())
+                .collect(Collectors.toList());
    }


-    public Float getHighest() {
+    public Double getHighest() {

-        Float highest = null;
-        for (Float value : countPerValue.keySet()) {
+        Double highest = null;
+        for (Double value : countPerValue.keySet()) {
            if (highest == null || value > highest) {
                highest = value;
            }
@@ -3,7 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,8 +16,8 @@ import lombok.experimental.FieldDefaults;
 public class LineInformation {

    List<Rectangle2D> lineBBox;
-    List<List<TextPositionSequence>> sequencesByLines;
+    List<List<Word>> sequencesByLines;
    List<List<Rectangle2D>> bBoxWithGapsByLines;
-    List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
+    List<List<List<Word>>> sequencesWithGapsByLines;

 }
@@ -9,12 +9,13 @@ public enum PageBlockType {
    H6,
    HEADER,
    FOOTER,
-    TITLE,
    PARAGRAPH,
    PARAGRAPH_BOLD,
    PARAGRAPH_ITALIC,
    PARAGRAPH_UNKNOWN,
    OTHER,
+    TABLE_OF_CONTENTS_ITEM,
+    LIST_ITEM,
    TABLE;


@@ -4,7 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -15,7 +15,7 @@ import lombok.Getter;
@AllArgsConstructor
 public class PageContents {

-    List<TextPositionSequence> sortedTextPositionSequences;
+    List<Word> sortedWords;
    Rectangle2D cropBox;
    Rectangle2D mediaBox;
    List<Ruling> rulings;
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

@@ -15,11 +16,13 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {

-    static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d{1,2})(?:[\\s.,;](\\d{1,2}))?(?:[\\s.,;](\\d{1,2}))?(?:[\\s.,;](\\d{1,2}))?");
+    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d{1,2})[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?");

    public enum Format {
        EMPTY,
        NUMERICAL,
+        ALPHANUMERIC,
        DOCUMENT
    }

@@ -41,6 +44,10 @@ public class SectionIdentifier {
        if (numericalIdentifierMatcher.find()) {
            return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher);
        }
+        Matcher alphanumericIdentifierMatcher = alphanumericIdentifierPattern.matcher(headline);
+        if (alphanumericIdentifierMatcher.find()) {
+            return buildAlphanumericSectionIdentifier(headline, alphanumericIdentifierMatcher);
+        }
        // more formats here
        return SectionIdentifier.empty();
    }
@@ -75,7 +82,36 @@ public class SectionIdentifier {
            }
            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
        }
-        return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false);
+        return new SectionIdentifier(Format.NUMERICAL,
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
+    }
+
+
+    private static SectionIdentifier buildAlphanumericSectionIdentifier(String headline, Matcher alphanumericIdentifierMatcher) {
+
+        String identifierString = headline.substring(alphanumericIdentifierMatcher.start(), alphanumericIdentifierMatcher.end());
+
+        String alphanumericIdentifier = alphanumericIdentifierMatcher.group(0).substring(0, 1).toUpperCase(Locale.ENGLISH);
+        int mappedCharacterValue = alphanumericIdentifier.charAt(0) - 'A' + 1;
+        List<Integer> identifiers = new LinkedList<>();
+        identifiers.add(mappedCharacterValue);
+
+        for (int i = 1; i <= 3; i++) {
+            String numericalIdentifier = alphanumericIdentifierMatcher.group(i);
+            if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) {
+                break;
+            }
+            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
+        }
+
+        return new SectionIdentifier(Format.ALPHANUMERIC,
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
    }


@@ -123,4 +159,22 @@ public class SectionIdentifier {
        return identifierString;
    }

+
+    public boolean isEmpty() {
+
+        return this.format.equals(Format.EMPTY);
+    }
+
+
+    public int level() {
+
+        return identifiers.size();
+    }
+
+
+    protected List<Integer> getIdentifiers() {
+
+        return identifiers;
+    }
+
 }
@@ -84,7 +84,7 @@ public abstract class AbstractNodeVisitor implements NodeVisitor {
    }


-    private void visitChildren(SemanticNode semanticNode) {
+    protected void visitChildren(SemanticNode semanticNode) {

        semanticNode.streamChildren()
                .forEach(node -> node.accept(this));
@@ -98,10 +98,10 @@ public class TextRange implements Comparable<TextRange> {
    public List<TextRange> split(List<Integer> splitIndices) {

        if (splitIndices.stream()
-                .anyMatch(idx -> !this.containsExclusive(idx))) {
+                .anyMatch(idx -> !this.contains(idx))) {
            throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s",
                                                       splitIndices.stream()
-                                                               .filter(idx -> !this.containsExclusive(idx))
+                                                               .filter(idx -> !this.contains(idx))
                                                               .toList(),
                                                       this));
        }
@@ -116,7 +116,9 @@ public class TextRange implements Comparable<TextRange> {
            splitBoundaries.add(new TextRange(previousIndex, splitIndex));
            previousIndex = splitIndex;
        }
-        splitBoundaries.add(new TextRange(previousIndex, end));
+        if (previousIndex != end) {
+            splitBoundaries.add(new TextRange(previousIndex, end));
+        }
        return splitBoundaries;
    }

@@ -25,11 +25,4 @@ public class DuplicatedParagraph extends Paragraph {

    }

-
-    @Override
-    public String toString() {
-
-        return super.toString();
-    }
-
 }
@@ -1,12 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Stream;

 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.TextEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;

@@ -29,9 +32,8 @@ public class Page {
    Integer height;
    Integer width;
    Integer rotation;
-
    @EqualsAndHashCode.Exclude
-    List<SemanticNode> mainBody;
+    List<AtomicTextBlock> textBlocksOnPage;
    @EqualsAndHashCode.Exclude
    Header header;
    @EqualsAndHashCode.Exclude
@@ -53,20 +55,44 @@ public class Page {
                .width((int) classificationPage.getPageWidth())
                .number(classificationPage.getPageNumber())
                .rotation(classificationPage.getRotation())
-                .mainBody(new LinkedList<>())
+                .textBlocksOnPage(new LinkedList<>())
                .build();
    }


+    /**
+     * Constructs and returns a {@link TextBlock} representing the concatenated text of all leaf semantic nodes in the main body.
+     *
+     * @return The main body text block.
+     */
    public TextBlock getMainBodyTextBlock() {

-        return mainBody.stream()
-                .filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getLeafTextBlock)
+        return textBlocksOnPage.stream()
+                .filter(atb -> !atb.isEmpty())
                .collect(new TextBlockCollector());
    }


+    public List<SemanticNode> getMainBody() {
+
+        return textBlocksOnPage.stream()
+                .map(AtomicTextBlock::getParent)
+                .map(this::getHighestParentOnPage)
+                .distinct()
+                .toList();
+    }
+
+
+    private SemanticNode getHighestParentOnPage(SemanticNode node) {
+
+        SemanticNode currentNode = node;
+        while (currentNode.hasParent() && currentNode.getParent().onlyOnPage(this)) {
+            currentNode = currentNode.getParent();
+        }
+        return currentNode;
+    }
+
+
    @Override
    public String toString() {

@@ -74,7 +74,8 @@ public interface SemanticNode {

        return getTextBlock().getPages()
                .stream()
-                .min(Comparator.comparingInt(Page::getNumber)).orElseThrow(() -> new IllegalStateException("SemanticNode has no Page!"));
+                .min(Comparator.comparingInt(Page::getNumber))
+                .orElseThrow(() -> new IllegalStateException("SemanticNode has no Page!"));
    }


@@ -504,4 +505,17 @@ public interface SemanticNode {

    void accept(NodeVisitor visitor);

+
+    /**
+     * Checks wether this SemanticNode appears on a single page only, and if that page is the provided one.
+     *
+     * @param page the page to check
+     * @return true, when SemanticNode is on a single page only and the page is the provided page. Otherwise, false.
+     */
+    default boolean onlyOnPage(Page page) {
+
+        Set<Page> pages = getPages();
+        return pages.size() == 1 && pages.contains(page);
+    }
+
 }
@@ -145,10 +145,7 @@ public class AtomicTextBlock implements TextBlock {
    }


-    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData,
-                                                          DocumentPositionData documentPositionData,
-                                                          SemanticNode parent,
-                                                          Page page) {
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData, DocumentPositionData documentPositionData, SemanticNode parent, Page page) {

        return AtomicTextBlock.builder()
                .id(documentTextData.getId())
@@ -156,8 +153,10 @@ public class AtomicTextBlock implements TextBlock {
                .page(page)
                .textRange(new TextRange(documentTextData.getStart(), documentTextData.getEnd()))
                .searchText(documentTextData.getSearchText())
-                .lineBreaks(Arrays.stream(documentTextData.getLineBreaks()).boxed().toList())
-                .stringIdxToPositionIdx(Arrays.stream(documentPositionData.getStringIdxToPositionIdx()).boxed().toList())
+                .lineBreaks(Arrays.stream(documentTextData.getLineBreaks()).boxed()
+                                    .toList())
+                .stringIdxToPositionIdx(Arrays.stream(documentPositionData.getStringIdxToPositionIdx()).boxed()
+                                                .toList())
                .positions(toRectangle2DList(documentPositionData.getPositions()))
                .parent(parent)
                .build();
@@ -166,7 +165,9 @@ public class AtomicTextBlock implements TextBlock {

    private static List<Rectangle2D> toRectangle2DList(float[][] positions) {

-        return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList();
+        return Arrays.stream(positions)
+                .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+                .toList();
    }


@@ -176,6 +177,9 @@ public class AtomicTextBlock implements TextBlock {
            throw new IndexOutOfBoundsException(format("line %d out of range for AtomicTextBlock with %d lines", lineNumber, numberOfLines()));
        }
        if (lineNumber == 0) {
+            if (lineBreaks.isEmpty()) {
+                return searchText;
+            }
            return subSequence(textRange.start(), lineBreaks.get(0) + textRange.start());
        } else if (lineNumber == numberOfLines() - 1) {
            return subSequence(lineBreaks.get(lineBreaks.size() - 1) + textRange.start(), textRange.end());
@@ -195,9 +199,9 @@ public class AtomicTextBlock implements TextBlock {
    public int getNextLinebreak(int fromIndex) {

        return lineBreaks.stream()//
-                .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
-                .findFirst() //
-                .orElse(searchText.length()) + textRange.start();
+                       .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
+                       .findFirst() //
+                       .orElse(searchText.length()) + textRange.start();
    }


@@ -205,9 +209,9 @@ public class AtomicTextBlock implements TextBlock {
    public int getPreviousLinebreak(int fromIndex) {

        return lineBreaks.stream()//
-                .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
-                .reduce((a, b) -> b)//
-                .orElse(0) + textRange.start();
+                       .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
+                       .reduce((a, b) -> b)//
+                       .orElse(0) + textRange.start();
    }


@@ -255,7 +259,10 @@ public class AtomicTextBlock implements TextBlock {

    protected List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {

-        return getLineBreaks().stream().map(linebreak -> linebreak + this.textRange.start()).filter(textRange::contains).toList();
+        return getLineBreaks().stream()
+                .map(linebreak -> linebreak + this.textRange.start())
+                .filter(textRange::contains)
+                .toList();
    }


@@ -10,8 +10,8 @@ import lombok.NonNull;
 import lombok.RequiredArgsConstructor;

@Data
-@RequiredArgsConstructor
@AllArgsConstructor
+@RequiredArgsConstructor
 public class ClassifiedImage {

    @NonNull
@@ -20,11 +20,18 @@ public class ClassifiedImage {
    private ImageType imageType;
    private boolean sourceByAi;
    private boolean isAppendedToSection;
-    @NonNull
    private boolean hasTransparency;
-    @NonNull
    private int page;
-    @NonNull
    private String representation;

+
+    public ClassifiedImage(@NonNull Rectangle2D position, @NonNull ImageType imageType, boolean hasTransparency, int page, String representation) {
+
+        this.position = position;
+        this.imageType = imageType;
+        this.hasTransparency = hasTransparency;
+        this.page = page;
+        this.representation = representation;
+    }
+
 }
@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;

+import java.awt.geom.AffineTransform;
 import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -26,6 +27,9 @@ import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocume
 import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@@ -82,11 +86,20 @@ public class OutlineExtractorService {

        String title = item.getTitle();

-        PDPage page = item.findDestinationPage(document);
-        if (page == null) {
+        PDPage page;
+        try {
+            // Can throw: "Error: can't convert to Destination COSArray" for some OCR'd PDFs
+            page = item.findDestinationPage(document);
+            if (page == null) {
+                return Optional.empty();
+            }
+        } catch (IOException e) {
+            log.info(String.format("Error occurred during position resolution for outline item with title %s: " + e, title));
            return Optional.empty();
        }
-        int pageNumber = document.getPages().indexOf(page);
+
+        int pageNumber = document.getPages().indexOf(page) + 1;
+        AffineTransform userSpaceToPageCoords = CoordinateTransforms.calculateInitialUserSpaceCoordsToPageCoords(PageInformation.fromPDPage(pageNumber, page));

        Optional<Point2D> outlinePosition = Optional.empty();

@@ -115,8 +128,15 @@ public class OutlineExtractorService {
            log.info(String.format("Error occurred during position resolution for outline item on page %s with title %s: " + e, pageNumber, title));
        }

-        return Optional.of(new OutlineObjectTreeNode(new OutlineObject(title, pageNumber, outlinePosition.orElse(new Point2D.Float(0, 0)), depth)));
+        return Optional.of(new OutlineObjectTreeNode(new OutlineObject(title,
+                                                                       pageNumber,
+                                                                       transformPointToPageCoords(outlinePosition, userSpaceToPageCoords), depth)));
+    }

+
+    private static Point2D transformPointToPageCoords(Optional<Point2D> outlinePosition, AffineTransform userSpaceToPageCoords) {
+
+        return outlinePosition.map(point -> userSpaceToPageCoords.transform(point, null)).orElse(null);
    }


@@ -1,27 +1,34 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;

 import java.awt.geom.Point2D;
+import java.util.Optional;

-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+
+import lombok.Getter;
+import lombok.Setter;

-@Data
-@RequiredArgsConstructor
-@AllArgsConstructor
 public class OutlineObject {

+    @Getter
    private final String title;
+    @Getter
    private final int pageNumber;
-    private Point2D point;
+    @Getter
    private final int treeDepth;

+    private Point2D point; // java coordinates, (0, 0) is always top left
+
+    @Getter
+    @Setter
    private boolean found;


    public OutlineObject(String title, int pageNumber, Point2D point2D, int depth) {

-        this(title, pageNumber, depth);
+        this.title = title;
+        this.pageNumber = pageNumber;
+        this.treeDepth = depth;
        this.point = point2D;
    }

@@ -32,4 +39,39 @@ public class OutlineObject {
        return "OutlineObject{" + "title='" + title + '\'' + '}';
    }

+
+    public Optional<Point2D> getPoint() {
+
+        return Optional.ofNullable(point);
+    }
+
+
+    public boolean isAbove(BoundingBox boundingBox) {
+
+        if (point == null) {
+            return true;
+        }
+        return point.getY() <= boundingBox.getMaxY();
+    }
+
+
+    public double distance(BoundingBox boundingBox) {
+
+        if (point == null) {
+            return 0;
+        }
+        if (boundingBox.getBBox().contains(point)) {
+            return 0;
+        }
+        double deltaX = Math.min(Math.abs(boundingBox.getMinX() - point.getX()), Math.abs(boundingBox.getMaxX() - point.getX()));
+        double deltaY = Math.min(Math.abs(boundingBox.getMinY() - point.getY()), Math.abs(boundingBox.getMaxY() - point.getY()));
+        return Math.sqrt(deltaX * deltaX + deltaY * deltaY);
+    }
+
+
+    public void resetPoint() {
+
+        this.point = null;
+    }
+
 }
@@ -39,4 +39,28 @@ public class OutlineObjectTree {
        }
    }

+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("OutlineObjectTree(\n");
+        for (OutlineObjectTreeNode node : rootNodes) {
+            buildString(node, sb, 1);
+        }
+        sb.append(")");
+        return sb.toString();
+    }
+
+    private void buildString(OutlineObjectTreeNode node, StringBuilder sb, int depth) {
+
+        for (int i = 0; i < depth; i++) {
+            sb.append("  ");
+        }
+        sb.append(node.getOutlineObject().getTitle()).append("\n");
+
+        for (OutlineObjectTreeNode child : node.getChildren()) {
+            buildString(child, sb, depth + 1);
+        }
+    }
+
 }
@@ -10,6 +10,7 @@ import java.util.TreeSet;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;

 import io.micrometer.observation.annotation.Observed;
@@ -20,7 +21,9 @@ import lombok.extern.slf4j.Slf4j;
 public class OutlineValidationService {

    @Observed(name = "OutlineValidationService", contextualName = "create-toc")
-    public TableOfContents createToC(List<TextPageBlock> headlines) {
+    public TableOfContents createToC(ClassificationDocument classificationDocument) {
+
+        List<TextPageBlock> headlines = extractHeadlines(classificationDocument);

        List<TableOfContentItem> mainSections = new ArrayList<>();
        Map<Integer, TableOfContentItem> lastItemsPerDepth = new HashMap<>();
@@ -60,4 +63,16 @@ public class OutlineValidationService {
        return new TableOfContents(mainSections);
    }

+
+    private static List<TextPageBlock> extractHeadlines(ClassificationDocument classificationDocument) {
+
+        return classificationDocument.getPages()
+                .stream()
+                .flatMap(classificationPage -> classificationPage.getTextBlocks()
+                        .stream()
+                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
+                        .map(tb -> (TextPageBlock) tb))
+                .toList();
+    }
+
 }
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;

 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -185,12 +186,8 @@ public class TOCEnrichmentService {
            List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
            List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
            // Allow merging of tables if header row is separated from first logical non-header row
-            if (previousTableNonHeaderRow.isEmpty()
-                && previousTable.getRowCount() == 1
-                && previousTable.getRows()
-                           .get(0).size() == tableNonHeaderRow.size()) {
-                previousTableNonHeaderRow = previousTable.getRows()
-                        .get(0)
+            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows().get(0).size() == tableNonHeaderRow.size()) {
+                previousTableNonHeaderRow = previousTable.getRows().get(0)
                        .stream()
                        .map(cell -> {
                            Cell fakeCell = Cell.copy(cell);
@@ -201,8 +198,7 @@ public class TOCEnrichmentService {
            }
            if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
                for (int i = currentTable.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-                    List<Cell> row = currentTable.getRows()
-                            .get(i);
+                    List<Cell> row = currentTable.getRows().get(i);
                    if (row.size() == tableNonHeaderRow.size() && row.stream()
                            .allMatch(cell -> cell.getHeaderCells().isEmpty())) {
                        for (int j = 0; j < row.size(); j++) {
@@ -225,18 +221,15 @@ public class TOCEnrichmentService {

        return table.getRows()
                .stream()
-                .flatMap(row -> row.stream()
-                        .filter(cell -> !cell.getHeaderCells().isEmpty()))
-                .findAny().isEmpty();
-
+                .flatMap(Collection::stream)
+                .allMatch(cell -> cell.getHeaderCells().isEmpty());
    }


    private List<Cell> getRowWithNonHeaderCells(TablePageBlock table) {

        for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-            List<Cell> row = table.getRows()
-                    .get(i);
+            List<Cell> row = table.getRows().get(i);
            if (row.size() == 1) {
                continue;
            }
@@ -5,7 +5,7 @@ import java.util.List;
 import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;

@@ -24,7 +24,7 @@ public class TableOfContentItem {
    private List<AbstractPageBlock> sectionBlocks = new ArrayList<>();
    private List<ClassifiedImage> images = new ArrayList<>();

-    private AbstractSemanticNode section;
+    private GenericSemanticNode section;


    public TableOfContentItem(TextPageBlock headline) {
@@ -45,8 +45,7 @@ public class TableOfContentItem {
        if (parent != null) {
            int index = parent.getChildren().indexOf(this);
            if (index > 0) {
-                return parent.getChildren()
-                        .get(index - 1);
+                return parent.getChildren().get(index - 1);
            }
        }
        return null;
@@ -58,8 +57,7 @@ public class TableOfContentItem {
        if (parent != null) {
            int index = parent.getChildren().indexOf(this);
            if (index >= 0 && index < parent.getChildren().size() - 1) {
-                return parent.getChildren()
-                        .get(index + 1);
+                return parent.getChildren().get(index + 1);
            }
        }
        return null;
@@ -93,17 +91,19 @@ public class TableOfContentItem {
        return false;
    }

+
    public List<AbstractPageBlock> getNonEmptySectionBlocks() {

-        return sectionBlocks.stream().filter(pageBlock -> !pageBlock.isEmpty()).collect(Collectors.toList());
+        return sectionBlocks.stream()
+                .filter(pageBlock -> !pageBlock.isEmpty())
+                .collect(Collectors.toList());
    }

+
    @Override
    public String toString() {

        return "OutlineObjectTreeNode{" + "textPageBlock=" + headline + '}';
    }

-
-
 }
@@ -9,7 +9,7 @@ import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.Data;
@@ -35,14 +35,14 @@ public class Cell extends BoundingBox {

    public Cell(Point2D topLeft, Point2D bottomRight) {

-        this.bBoxInitialUserSpace = new Rectangle2D.Double(topLeft.getX(), topLeft.getY(), (bottomRight.getX() - topLeft.getX()), (bottomRight.getY() - topLeft.getY()));
-        this.bBox = bBoxInitialUserSpace;
+        this.bBoxPdf = new Rectangle2D.Double(topLeft.getX(), topLeft.getY(), (bottomRight.getX() - topLeft.getX()), (bottomRight.getY() - topLeft.getY()));
+        this.bBox = bBoxPdf;
    }


    public Cell(Rectangle2D bBoxInitialUserSpace, AffineTransform initialUserSpaceToJava) {

-        this.bBoxInitialUserSpace = bBoxInitialUserSpace;
+        this.bBoxPdf = bBoxInitialUserSpace;
        this.bBox = initialUserSpaceToJava.createTransformedShape(bBoxInitialUserSpace).getBounds2D();
    }

@@ -50,7 +50,7 @@ public class Cell extends BoundingBox {
    public static Cell copy(Cell cell) {

        Cell copy = new Cell();
-        copy.bBoxInitialUserSpace = cell.bBoxInitialUserSpace;
+        copy.bBoxPdf = cell.bBoxPdf;
        copy.bBox = cell.bBox;
        return copy;
    }
@@ -68,12 +68,12 @@ public class Cell extends BoundingBox {
        StringBuilder sb = new StringBuilder();

        Iterator<TextPageBlock> itty = textBlocks.iterator();
-        TextPositionSequence previous = null;
+        Word previous = null;
        while (itty.hasNext()) {

            TextPageBlock textBlock = itty.next();

-            for (TextPositionSequence word : textBlock.getSequences()) {
+            for (Word word : textBlock.getWords()) {
                if (previous != null) {
                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                        sb.append('\n');
@@ -87,7 +87,7 @@ public class Cell extends BoundingBox {

        }

-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
+        return TextNormalizationUtilities.cleanString(sb.toString());
    }


@@ -70,7 +70,7 @@ public class CleanRulings {

    public boolean lineBetween(BoundingBox a, BoundingBox b) {

-        return lineBetween(a.getBBoxInitialUserSpace(), b.getBBoxInitialUserSpace());
+        return lineBetween(a.getBBoxPdf(), b.getBBoxPdf());
    }


@@ -263,8 +263,8 @@ public class TablePageBlock extends AbstractPageBlock {

                        cells.stream()
                                .map(originalCell -> new CellWithIntersection(originalCell,
-                                                                              RectangleTransformations.calculateIntersectedArea(cellFromGridStructure.getBBoxInitialUserSpace(),
-                                                                                                                                originalCell.getBBoxInitialUserSpace())))
+                                                                              RectangleTransformations.calculateIntersectedArea(cellFromGridStructure.getBBoxPdf(),
+                                                                                                                                originalCell.getBBoxPdf())))
                                .filter(cellWithIntersection -> cellWithIntersection.intersectedArea > 0)
                                .filter(cellWithIntersection -> cellWithIntersection.originalCell.getArea() > cellWithIntersection.intersectedArea * CELL_AREA_CONTAINED_THRESHOLD)
                                .max(Comparator.comparing(CellWithIntersection::intersectedArea))
@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+
+public record AbstractBlockOnPage(AbstractPageBlock block, ClassificationPage page) {
+
+}
@@ -0,0 +1,9 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+
+public enum FontStyle {
+    REGULAR,
+    BOLD,
+    ITALIC,
+    BOLD_ITALIC;
+}
@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@NoArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FrequencyCounters {
+
+    FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
+    FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
+    FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
+    StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
+    StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
+
+}
@@ -0,0 +1,107 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class ListIdentifier {
+
+    public static final Pattern NUMBER_WITH_DOT = Pattern.compile("^\\s*([1-9]{1,4})\\.\\s+");
+    public static final Pattern NUMBER_IN_PARENTHESES = Pattern.compile("^\\s*\\(([1-9]{1,4})\\)\\s+");
+
+    enum Format {
+        NUMBER_WITH_DOT,
+        NUMBER_IN_PARENTHESES
+    }
+
+    Format format;
+    @Getter
+    Word word;
+    @Getter
+    int page;
+    int representation;
+
+
+    public static Optional<ListIdentifier> parse(TextPageBlock textPageBlock, int page) {
+
+        return parse(textPageBlock.getWords().subList(0, Math.min(5, textPageBlock.getWords().size())), page);
+    }
+
+
+    public static Optional<ListIdentifier> parse(List<Word> sequences, int page) {
+
+        StringBuilder sb = new StringBuilder();
+        for (Word sequence : sequences) {
+            sb.append(sequence.toString());
+            sb.append(" ");
+        }
+        sb.replace(sb.length() - 1, sb.length(), "");
+        String text = sb.toString();
+
+        Matcher numberMatcher = NUMBER_WITH_DOT.matcher(text);
+
+        if (numberMatcher.find()) {
+            Optional<Integer> representation = parseInteger(numberMatcher.group(1));
+            if (representation.isPresent()) {
+                return Optional.of(new ListIdentifier(Format.NUMBER_WITH_DOT, sequences.get(0), page, representation.get()));
+            }
+        }
+
+        Matcher parenthesisMatcher = NUMBER_IN_PARENTHESES.matcher(text);
+        if (parenthesisMatcher.find()) {
+            Optional<Integer> representation = parseInteger(parenthesisMatcher.group(1));
+            if (representation.isPresent()) {
+                return Optional.of(new ListIdentifier(Format.NUMBER_IN_PARENTHESES, sequences.get(0), page, representation.get()));
+            }
+        }
+        return Optional.empty();
+    }
+
+
+    private static Optional<Integer> parseInteger(String text) {
+
+        try {
+            return Optional.of(Integer.parseInt(text));
+        } catch (NumberFormatException e) {
+            return Optional.empty();
+        }
+    }
+
+
+    public static boolean isInOrder(List<ListIdentifier> listIdentifiers) {
+
+        if (listIdentifiers.size() <= 1) {
+            return true;
+        }
+
+        for (int i = 1; i < listIdentifiers.size(); i++) {
+            ListIdentifier current = listIdentifiers.get(i);
+            ListIdentifier previous = listIdentifiers.get(i - 1);
+            if (current.format != previous.format) {
+                return false;
+            }
+            if (current.representation <= previous.representation) {
+                return false;
+            }
+            if (!current.word.intersectsXDirAdj(previous.word, 2)) {
+                return false;
+            }
+            if (current.page == previous.page && !current.word.isBelowDirAdj(previous.word)) {
+                return false;
+            }
+            if (current.page < previous.page) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+}
@@ -5,64 +5,52 @@ import java.awt.geom.Rectangle2D;

 import org.apache.pdfbox.text.TextPosition;

-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;

+import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;

@Data
-@Builder
+@SuperBuilder
@NoArgsConstructor
@AllArgsConstructor
-public class RedTextPosition extends BoundingBox {
+@EqualsAndHashCode(callSuper = true)
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class RedTextPosition extends TextBoundingBox {

    public final static int HEIGHT_PADDING = 2;

-    private Rectangle2D.Float bBoxDirAdj; // adjusted to text rotation
+    String unicode;

-    @JsonIgnore
-    private int rotation;
+    // estimated using the TextMatrix in radians
+    float exactDir;

-    @JsonIgnore
-    private float pageHeight;
+    float widthOfSpace;

-    @JsonIgnore
-    private float pageWidth;
+    float fontSizeInPt;

-    private String unicode;
-
-    @JsonIgnore
-    private float dir;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private float widthOfSpace;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private float fontSizeInPt;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private String fontName;
+    String fontName;


    @SneakyThrows
    public static RedTextPosition fromTextPosition(TextPosition textPosition) {

        var pos = new RedTextPosition();
-        pos.setRotation(textPosition.getRotation());
-        pos.setPageHeight(textPosition.getPageHeight());
-        pos.setPageWidth(textPosition.getPageWidth());
        pos.setUnicode(textPosition.getUnicode());
-        pos.setDir(textPosition.getDir());
        pos.setWidthOfSpace(textPosition.getWidthOfSpace());
        pos.setFontSizeInPt(textPosition.getFontSizeInPt());
        pos.setFontName(textPosition.getFont().getName());
+        pos.setExactDir((float) FastAtan2.fastAtan2(textPosition.getTextMatrix().getShearY(), textPosition.getTextMatrix().getScaleX()));
+        pos.setDir(TextDirection.fromDegrees(textPosition.getDir()));

        //TODO: There is a mismatch in the java coords of the text and the rulings,
        // I guess if we start with the initial user space positions and transform them the same way we do the rulings it would work.
@@ -73,18 +61,18 @@ public class RedTextPosition extends BoundingBox {
                                                                 textPosition.getYDirAdj() - textHeight,
                                                                 textPosition.getWidthDirAdj(),
                                                                 textHeight + HEIGHT_PADDING);
+
        pos.setBBoxDirAdj(dirAdjPosition);

        AffineTransform affineTransform = getRotationMatrix(TextDirection.fromDegrees(textPosition.getDir()), textPosition.getPageWidth(), textPosition.getPageHeight());
        Rectangle2D bBoxInitialUserSpace = affineTransform.createTransformedShape(dirAdjPosition).getBounds2D();

-        pos.setBBoxInitialUserSpace(bBoxInitialUserSpace); // These are definitely correct
+        pos.setBBoxPdf(bBoxInitialUserSpace); // These are definitely correct

        return pos;
    }


-
    private static AffineTransform getRotationMatrix(TextDirection textDirection, float pageWidth, float pageHeight) {

        AffineTransform transform = new AffineTransform();
@@ -103,32 +91,4 @@ public class RedTextPosition extends BoundingBox {
        return transform;
    }

-
-    @JsonIgnore
-    public float getXDirAdj() {
-
-        return this.bBoxDirAdj.x;
-    }
-
-
-    @JsonIgnore
-    public float getYDirAdj() {
-
-        return this.bBoxDirAdj.y;
-    }
-
-
-    @JsonIgnore
-    public float getWidthDirAdj() {
-
-        return this.bBoxDirAdj.width;
-    }
-
-
-    @JsonIgnore
-    public float getHeightDir() {
-
-        return this.bBoxDirAdj.height;
-    }
-
 }
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import java.util.ArrayList;
 import java.util.List;
+
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.Getter;
@@ -9,18 +10,18 @@ import lombok.Getter;
@Getter
 public class SearchableText {

-    private final List<TextPositionSequence> sequences = new ArrayList<>();
+    private final List<Word> sequences = new ArrayList<>();


-    public void add(TextPositionSequence textPositionSequence) {
+    public void add(Word word) {

-        sequences.add(textPositionSequence);
+        sequences.add(word);
    }


-    public void addAll(List<TextPositionSequence> textPositionSequences) {
+    public void addAll(List<Word> words) {

-        sequences.addAll(textPositionSequences);
+        sequences.addAll(words);
    }


@@ -31,18 +32,14 @@ public class SearchableText {
    }


-    public static String buildString(List<TextPositionSequence> sequences) {
+    public static String buildString(List<Word> sequences) {

        StringBuilder sb = new StringBuilder();
-        for (TextPositionSequence word : sequences) {
+        for (Word word : sequences) {
            sb.append(word);
            sb.append(' ');
        }
-        String text = sb.toString();
-        text = TextNormalizationUtilities.removeHyphenLineBreaks(text);
-        text = TextNormalizationUtilities.removeLineBreaks(text);
-        text = TextNormalizationUtilities.removeRepeatingWhitespaces(text);
-        return text;
+        return TextNormalizationUtilities.cleanString(sb.toString());
    }

 }
@@ -9,10 +9,14 @@ public class StringFrequencyCounter {

    @Getter
    private final Map<String, Integer> countPerValue = new HashMap<>();
+    boolean changed;
+    String mostPopularCache;


    public void add(String value) {

+        changed = true;
+
        if (!countPerValue.containsKey(value)) {
            countPerValue.put(value, 1);
        } else {
@@ -23,6 +27,8 @@ public class StringFrequencyCounter {

    public void addAll(Map<String, Integer> otherCounter) {

+        changed = true;
+
        for (Map.Entry<String, Integer> entry : otherCounter.entrySet()) {
            if (countPerValue.containsKey(entry.getKey())) {
                countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
@@ -35,13 +41,18 @@ public class StringFrequencyCounter {

    public String getMostPopular() {

-        Map.Entry<String, Integer> mostPopular = null;
-        for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
-                mostPopular = entry;
+        if (changed || mostPopularCache == null) {
+            Map.Entry<String, Integer> mostPopular = null;
+            for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
+                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
+                    mostPopular = entry;
+                }
            }
+            mostPopularCache = mostPopular != null ? mostPopular.getKey() : null;
+            changed = false;
        }
-        return mostPopular != null ? mostPopular.getKey() : null;
+
+        return mostPopularCache;
    }

 }
@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+
+public record TextBlockOnPage(TextPageBlock textBlock, ClassificationPage page) {
+
+}
@@ -44,4 +44,15 @@ public enum TextDirection {

        throw new IllegalArgumentException(String.format("A value of %f is not supported by TextDirection", degrees));
    }
+
+
+    public int getRotation() {
+
+        return switch (this) {
+            case ZERO -> 0;
+            case QUARTER_CIRCLE -> 1;
+            case HALF_CIRCLE -> 2;
+            case THREE_QUARTER_CIRCLE -> 3;
+        };
+    }
 }
@@ -2,12 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;

 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.AllArgsConstructor;
@@ -24,62 +25,59 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {

    @Builder.Default
-    private List<TextPositionSequence> sequences = new ArrayList<>();
+    private List<Word> words = new ArrayList<>();
+    @Builder.Default
+    private FrequencyCounters frequencyCounters = new FrequencyCounters();

-    private String mostPopularWordFont;
+    private Rectangle2D bBoxDirAdj;

-    private String mostPopularWordStyle;
-
-    private float mostPopularWordFontSize;
-
-    private float mostPopularWordHeight;
-
-    private float mostPopularWordSpaceWidth;
-
-    private float highestFontSize;
+    private boolean underlined;

    private PageBlockType classification;

    private boolean toDuplicate;

+    private String text;
+    private boolean changed;

-    public TextPageBlock(List<TextPositionSequence> sequences) {

-        this.sequences = sequences;
-        calculateFrequencyCounters();
+    public TextPageBlock(List<Word> words) {
+
+        this.words = new ArrayList<>(words);
+        this.frequencyCounters = new FrequencyCounters();
+
+        if (!words.isEmpty()) {
+            addToFrequencyCounters(words);
+        }
        calculateBBox();
    }


-    @JsonIgnore
+    public List<Word> getWords() {
+
+        return Collections.unmodifiableList(words);
+    }
+
+
    public TextDirection getDir() {

-        return sequences.get(0).getDir();
-    }
-
-
-    @JsonIgnore
-    public float getPageHeight() {
-
-        return sequences.get(0).getPageHeight();
-    }
-
-
-    @JsonIgnore
-    public float getPageWidth() {
-
-        return sequences.get(0).getPageWidth();
+        return words.get(0).getDir();
    }


    private void calculateBBox() {

-        if (sequences == null) {
+        if (words == null) {
            this.bBox = new Rectangle2D.Double();
-            this.bBoxInitialUserSpace = new Rectangle2D.Double();
+            this.bBoxPdf = new Rectangle2D.Double();
+            this.bBoxDirAdj = new Rectangle2D.Double();
            return;
        }
-        setToBBoxOfComponents(sequences);
+        this.bBoxDirAdj = words.stream()
+                .map(Word::getBBoxDirAdj)
+                .collect(RectangleTransformations.collectBBox());
+
+        setToBBoxOfComponents(words);
    }


@@ -101,8 +99,8 @@ public class TextPageBlock extends AbstractPageBlock {
            throw new IllegalArgumentException("Cannot merge textBlocks on different pages.");
        }

-        List<TextPositionSequence> sequences = textBlocksToMerge.stream()
-                .map(TextPageBlock::getSequences)
+        List<Word> sequences = textBlocksToMerge.stream()
+                .map(TextPageBlock::getWords)
                .flatMap(java.util.Collection::stream)
                .toList();
        sequences = new ArrayList<>(sequences);
@@ -111,38 +109,27 @@ public class TextPageBlock extends AbstractPageBlock {
    }


-    private void calculateFrequencyCounters() {
+    private void addToFrequencyCounters(List<Word> sequences) {

-        FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
-        FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
-        FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
-        StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
-        StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
-
-        for (TextPositionSequence wordBlock : sequences) {
-
-            lineHeightFrequencyCounter.add(wordBlock.getTextHeight());
-            fontSizeFrequencyCounter.add(wordBlock.getFontSize());
-            spaceFrequencyCounter.add(wordBlock.getSpaceWidth());
-            fontFrequencyCounter.add(wordBlock.getFont());
-            styleFrequencyCounter.add(wordBlock.getFontStyle());
+        for (Word wordBlock : sequences) {

+            frequencyCounters.getLineHeightFrequencyCounter().add(wordBlock.getTextHeight());
+            frequencyCounters.getFontSizeFrequencyCounter().add(wordBlock.getFontSize());
+            frequencyCounters.getSpaceFrequencyCounter().add(wordBlock.getSpaceWidth());
+            frequencyCounters.getFontFrequencyCounter().add(wordBlock.getFont());
+            frequencyCounters.getStyleFrequencyCounter().add(wordBlock.getFontStyle());
        }

-        setMostPopularWordFont(fontFrequencyCounter.getMostPopular());
-        setMostPopularWordStyle(styleFrequencyCounter.getMostPopular());
-        setMostPopularWordFontSize(fontSizeFrequencyCounter.getMostPopular());
-        setMostPopularWordHeight(lineHeightFrequencyCounter.getMostPopular());
-        setMostPopularWordSpaceWidth(spaceFrequencyCounter.getMostPopular());
-        setHighestFontSize(fontSizeFrequencyCounter.getHighest());
+        setUnderlined(this.words.stream()
+                              .allMatch(Word::isUnderline));
    }


-    public TextPageBlock union(TextPositionSequence r) {
+    public TextPageBlock union(Word r) {

        TextPageBlock union = this.copy();
-        union.getSequences().add(r);
-        calculateFrequencyCounters();
+        union.add(r);
+        addToFrequencyCounters(List.of(r));
        calculateBBox();
        return union;
    }
@@ -151,51 +138,50 @@ public class TextPageBlock extends AbstractPageBlock {
    public TextPageBlock union(TextPageBlock r) {

        TextPageBlock union = this.copy();
-        union.getSequences().addAll(r.getSequences());
-        calculateFrequencyCounters();
+        union.addAll(r.getWords());
+        addToFrequencyCounters(r.getWords());
        calculateBBox();
        return union;
    }


-    public void add(TextPageBlock r) {
+    public void add(TextPageBlock textPageBlock) {

-        sequences.addAll(r.getSequences());
-        calculateFrequencyCounters();
+        changed = true;
+        words.addAll(textPageBlock.getWords());
+        addToFrequencyCounters(textPageBlock.getWords());
        calculateBBox();
    }


-    public void add(TextPositionSequence r) {
+    public void add(Word word) {

-        sequences.add(r);
-        calculateFrequencyCounters();
+        changed = true;
+        words.add(word);
+        addToFrequencyCounters(List.of(word));
+        calculateBBox();
+    }
+
+
+    public void addAll(List<Word> words) {
+
+        changed = true;
+        this.words.addAll(words);
+        addToFrequencyCounters(words);
        calculateBBox();
    }


    public TextPageBlock copy() {

-        return new TextPageBlock(new ArrayList<>(sequences));
+        return new TextPageBlock(new ArrayList<>(words));
    }


    @Override
    public String toString() {

-        StringBuilder builder = new StringBuilder();
-
-        for (int i = 0; i < sequences.size(); i++) {
-            String sequenceAsString = sequences.get(i).toString();
-            // Fix for missing Whitespace. This is recognized in getSequences method. See PDFTextStripper Line 1730.
-            if (i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() - 1) != ' ' && sequenceAsString.charAt(0) != ' ') {
-                builder.append(' ');
-            }
-            builder.append(sequenceAsString);
-        }
-
-        return builder.toString();
-
+        return getText();
    }


@@ -203,30 +189,36 @@ public class TextPageBlock extends AbstractPageBlock {
    @JsonIgnore
    public String getText() {

-        StringBuilder sb = new StringBuilder();
+        if (text == null || changed) {

-        TextPositionSequence previous = null;
-        for (TextPositionSequence word : sequences) {
-            if (previous != null) {
-                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
-                    sb.append('\n');
-                } else {
-                    sb.append(' ');
+            StringBuilder sb = new StringBuilder();
+
+            Word previous = null;
+            for (Word word : words) {
+                if (previous != null) {
+                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
+                        sb.append('\n');
+                    } else {
+                        sb.append(' ');
+                    }
                }
+                sb.append(word.toString());
+                previous = word;
            }
-            sb.append(word.toString());
-            previous = word;
+
+            text = TextNormalizationUtilities.removeHyphenLinebreaks(sb.toString());
+            changed = false;
        }

-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString());
+        return text;
    }


    public int getNumberOfLines() {

        int numberOfLines = 1;
-        TextPositionSequence previous = null;
-        for (TextPositionSequence word : sequences) {
+        Word previous = null;
+        for (Word word : words) {
            if (previous != null) {
                if (word.getMaxYDirAdj() - previous.getMaxYDirAdj() > word.getTextHeight()) {
                    numberOfLines++;
@@ -238,10 +230,47 @@ public class TextPageBlock extends AbstractPageBlock {
    }


+    public String getMostPopularWordFont() {
+
+        return frequencyCounters.getFontFrequencyCounter().getMostPopular();
+    }
+
+
+    public String getMostPopularWordStyle() {
+
+        return frequencyCounters.getStyleFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordFontSize() {
+
+        return frequencyCounters.getFontSizeFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordHeight() {
+
+        return frequencyCounters.getLineHeightFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordSpaceWidth() {
+
+        return frequencyCounters.getSpaceFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getHighestFontSize() {
+
+        Double highest = frequencyCounters.getFontSizeFrequencyCounter().getHighest();
+        return highest == null ? 0 : highest;
+    }
+
+
    @Override
    public boolean isEmpty() {

-        return sequences.isEmpty();
+        return words.isEmpty();
    }

 }
@@ -1,271 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.stream.Collectors;
-
-import org.apache.pdfbox.text.TextPosition;
-
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
-public class TextPositionSequence extends BoundingBox implements CharSequence {
-
-    public static final int HEIGHT_PADDING = 2;
-
-    @EqualsAndHashCode.Include
-    private int page;
-
-    @EqualsAndHashCode.Include
-    private List<RedTextPosition> textPositions = new ArrayList<>();
-
-    private Rectangle2D bBoxDirAdj;
-    @EqualsAndHashCode.Include
-    private TextDirection dir;
-    private int rotation;
-    private float pageHeight;
-    private float pageWidth;
-    private boolean isParagraphStart;
-    private boolean strikethrough;
-    private boolean underline;
-
-
-    public TextPositionSequence(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
-
-        this.textPositions = textPositions.stream()
-                .map(RedTextPosition::fromTextPosition)
-                .collect(Collectors.toList());
-        this.page = pageNumber;
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        this.isParagraphStart = isParagraphStart;
-        calculateBBox();
-    }
-
-
-    private void calculateBBox() {
-
-        this.bBoxDirAdj = textPositions.stream()
-                .map(RedTextPosition::getBBoxDirAdj)
-                .collect(RectangleTransformations.collectBBox());
-        setToBBoxOfComponents(getTextPositions());
-    }
-
-
-    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
-
-        this.textPositions = textPositions;
-        this.page = page;
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        calculateBBox();
-    }
-
-
-    @Override
-    public int length() {
-
-        return textPositions.size();
-    }
-
-
-    @Override
-    public char charAt(int index) {
-
-        RedTextPosition textPosition = textPositionAt(index);
-        String text = textPosition.getUnicode();
-        return text.charAt(0);
-    }
-
-
-    public char charAt(int index, boolean caseInSensitive) {
-
-        RedTextPosition textPosition = textPositionAt(index);
-        String text = textPosition.getUnicode();
-        return caseInSensitive ? text.toLowerCase(Locale.ROOT).charAt(0) : text.charAt(0);
-    }
-
-
-    @Override
-    public TextPositionSequence subSequence(int start, int end) {
-
-        var textPositionSequence = new TextPositionSequence();
-        textPositionSequence.textPositions = textPositions.subList(start, end);
-        textPositionSequence.page = page;
-        textPositionSequence.dir = dir;
-        textPositionSequence.rotation = rotation;
-        textPositionSequence.pageHeight = pageHeight;
-        textPositionSequence.pageWidth = pageWidth;
-        textPositionSequence.setToBBoxOfComponents(getTextPositions());
-        return textPositionSequence;
-    }
-
-
-    @Override
-    public String toString() {
-
-        StringBuilder builder = new StringBuilder(length());
-        for (int i = 0; i < length(); i++) {
-            builder.append(charAt(i));
-        }
-        return builder.toString();
-    }
-
-
-    public RedTextPosition textPositionAt(int index) {
-
-        return textPositions.get(index);
-    }
-
-
-    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {
-
-        this.textPositions.add(textPosition);
-        this.page = textPositionSequence.getPage();
-        this.dir = textPositionSequence.getDir();
-        this.rotation = textPositionSequence.getRotation();
-        this.pageHeight = textPositionSequence.getPageHeight();
-        this.pageWidth = textPositionSequence.getPageWidth();
-        calculateBBox();
-    }
-
-
-    public void add(TextPosition textPosition) {
-
-        this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        calculateBBox();
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted minX value
-     */
-
-    public float getMinXDirAdj() {
-
-        return textPositions.get(0).getXDirAdj();
-
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted maxX value
-     */
-
-    public float getMaxXDirAdj() {
-
-        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
-
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
-     */
-
-    public float getMinYDirAdj() {
-
-        return textPositions.get(0).getYDirAdj() - getTextHeight();
-
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
-     */
-
-    public float getMaxYDirAdj() {
-
-        return textPositions.get(0).getYDirAdj();
-
-    }
-
-
-    public float getTextHeightNoPadding() {
-
-        return textPositions.get(0).getHeightDir();
-    }
-
-
-    public float getTextHeight() {
-
-        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
-    }
-
-
-    public String getFont() {
-
-        if (textPositions.get(0).getFontName() == null) {
-            return "none";
-        }
-        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
-    }
-
-
-    public String getFontStyle() {
-
-        if (textPositions.get(0).getFontName() == null) {
-            return "standard";
-        }
-        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);
-
-        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
-            return "bold, italic";
-        } else if (lowercaseFontName.contains("bold")) {
-            return "bold";
-        } else if (lowercaseFontName.contains("italic")) {
-            return "italic";
-        } else {
-            return "standard";
-        }
-    }
-
-
-    public float getFontSize() {
-
-        return textPositions.get(0).getFontSizeInPt();
-    }
-
-
-    public float getSpaceWidth() {
-
-        return textPositions.get(0).getWidthOfSpace();
-    }
-
-}
-
@@ -0,0 +1,36 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import java.util.Comparator;
+import java.util.HashMap;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.NumberWord;
+
+public class TocNumberComparator implements Comparator<NumberWord> {
+
+    private HashMap<NumberWord, TextBlockOnPage> lookup;
+
+
+    public TocNumberComparator(HashMap<NumberWord, TextBlockOnPage> lookup) {
+
+        this.lookup = lookup;
+    }
+
+
+    @Override
+    public int compare(NumberWord number1, NumberWord number2) {
+
+        int page1 = lookup.get(number1).page().getPageNumber();
+        int page2 = lookup.get(number2).page().getPageNumber();
+
+        if (page1 != page2) {
+            return Integer.compare(page1, page2);
+        }
+
+        if (number1.word().getY() != number2.word().getY()) {
+            return Double.compare(number1.word().getY(), number2.word().getY());
+        }
+
+        return Integer.compare(number1.number(), number2.number());
+    }
+
+}
@@ -0,0 +1,257 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition.HEIGHT_PADDING;
+
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.text.TextPosition;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@SuppressWarnings("pmd")
+public class Word extends TextBoundingBox implements CharSequence {
+
+    public static final String STANDARD = "standard";
+    public static final String BOLD_ITALIC = "bold, italic";
+    public static final String BOLD = "bold";
+    public static final String ITALIC = "italic";
+    public static final Pattern FONT_CLEANER = Pattern.compile(",bold|,italic");
+
+    private int page;
+
+    @Builder.Default
+    private List<RedTextPosition> textPositions = new ArrayList<>();
+
+    private boolean isParagraphStart;
+    private boolean strikethrough;
+    private boolean underline;
+
+    private Integer hashcodeCache;
+
+
+    public Word(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
+
+        this.textPositions = textPositions.stream()
+                .map(RedTextPosition::fromTextPosition)
+                .collect(Collectors.toList());
+        this.page = pageNumber;
+        this.isParagraphStart = isParagraphStart;
+        calculateBBoxAndHashcode();
+    }
+
+
+    private void calculateBBoxAndHashcode() {
+
+        setToBBoxOfComponents(getTextPositions());
+        hashcodeCache = null;
+    }
+
+
+    public Word(List<RedTextPosition> textPositions, int page) {
+
+        this.textPositions = textPositions;
+        this.page = page;
+        calculateBBoxAndHashcode();
+    }
+
+
+    @Override
+    public int length() {
+
+        return textPositions.size();
+    }
+
+
+    @Override
+    public char charAt(int index) {
+
+        RedTextPosition textPosition = textPositionAt(index);
+        String text = textPosition.getUnicode();
+        return text.charAt(0);
+    }
+
+
+    public char charAt(int index, boolean caseInSensitive) {
+
+        RedTextPosition textPosition = textPositionAt(index);
+        String text = textPosition.getUnicode();
+        return caseInSensitive ? text.toLowerCase(Locale.ROOT).charAt(0) : text.charAt(0);
+    }
+
+
+    @Override
+    public Word subSequence(int start, int end) {
+
+        var textPositionSequence = new Word();
+        textPositionSequence.textPositions = textPositions.subList(start, end);
+        textPositionSequence.page = page;
+        textPositionSequence.dir = dir;
+        textPositionSequence.setToBBoxOfComponents(getTextPositions());
+        return textPositionSequence;
+    }
+
+
+    @Override
+    public String toString() {
+
+        StringBuilder builder = new StringBuilder(length());
+        for (int i = 0; i < length(); i++) {
+            builder.append(charAt(i));
+        }
+        return builder.toString();
+    }
+
+
+    public RedTextPosition textPositionAt(int index) {
+
+        return textPositions.get(index);
+    }
+
+
+    public void add(Word word, RedTextPosition textPosition) {
+
+        this.textPositions.add(textPosition);
+        this.page = word.getPage();
+        calculateBBoxAndHashcode();
+    }
+
+
+    public void add(TextPosition textPosition) {
+
+        this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
+        calculateBBoxAndHashcode();
+    }
+
+
+    public double getTextHeightNoPadding() {
+
+        return textPositions.get(0).getHeightDirAdj();
+    }
+
+
+    public double getTextHeight() {
+
+        return textPositions.get(0).getHeightDirAdj() + HEIGHT_PADDING;
+    }
+
+
+    public String getFont() {
+
+        if (textPositions.get(0).getFontName() == null) {
+            return "none";
+        }
+
+        return FONT_CLEANER.matcher(textPositions.get(0).getFontName().toLowerCase(Locale.ROOT)).replaceAll("");
+    }
+
+
+    public String getFontStyle() {
+
+        if (textPositions.get(0).getFontName() == null) {
+            return STANDARD;
+        }
+        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);
+
+        if (lowercaseFontName.contains(BOLD) && lowercaseFontName.contains(ITALIC)) {
+            return BOLD_ITALIC;
+        } else if (lowercaseFontName.contains(BOLD)) {
+            return BOLD;
+        } else if (lowercaseFontName.contains(ITALIC)) {
+            return ITALIC;
+        } else {
+            return STANDARD;
+        }
+    }
+
+
+    public float getFontSize() {
+
+        return textPositions.get(0).getFontSizeInPt();
+    }
+
+
+    public float getSpaceWidth() {
+
+        return textPositions.get(0).getWidthOfSpace();
+    }
+
+
+    public boolean equals(final Object o) {
+        // auto-generated with lombok
+        if (o == this) {
+            return true;
+        }
+        if (!(o instanceof Word other)) {
+            return false;
+        }
+        if (!other.canEqual((Object) this)) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        if (this.getPage() != other.getPage()) {
+            return false;
+        }
+        final Object this$textPositions = this.getTextPositions();
+        final Object other$textPositions = other.getTextPositions();
+        if (!Objects.equals(this$textPositions, other$textPositions)) {
+            return false;
+        }
+        return Objects.equals(this.getHashcodeCache(), other.getHashcodeCache());
+    }
+
+
+    protected boolean canEqual(final Object other) {return other instanceof Word;}
+
+
+    public int hashCode() {
+
+        if (hashcodeCache == null) {
+            hashcodeCache = hashcodeCalculation();
+        }
+
+        return hashcodeCache;
+    }
+
+
+    private int hashcodeCalculation() {
+
+        final int PRIME = 59;
+        int result = super.hashCode();
+        result = result * PRIME + this.getPage();
+        final Object $textPositions = this.getTextPositions();
+        result = result * PRIME + ($textPositions == null ? 43 : $textPositions.hashCode());
+        return result;
+    }
+
+
+    public void transform(AffineTransform rotateInstance) {
+
+        for (RedTextPosition textPosition : getTextPositions()) {
+            Rectangle2D exactDirAdjCoordinates = rotateInstance.createTransformedShape(textPosition.getBBoxDirAdj()).getBounds2D();
+            textPosition.setBBoxDirAdj(exactDirAdjCoordinates);
+        }
+        calculateBBoxAndHashcode();
+    }
+
+}
+
@@ -56,7 +56,7 @@ public class ImageServiceResponseAdapter {
        classificationPage.getImages().forEach(image -> {
            if (image.getImageType().equals(ImageType.OTHER)) {
                for (AbstractPageBlock textblock : classificationPage.getTextBlocks()) {
-                    if (image.getPosition().contains(textblock.getBBoxInitialUserSpace())) {
+                    if (image.getPosition().contains(textblock.getBBoxPdf())) {
                        image.setImageType(ImageType.OCR);
                        return;
                    }
@@ -14,6 +14,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
 public class Classification {

+    @Builder.Default
    private Map<String, Float> probabilities = new HashMap<>();
    private String label;

@@ -22,8 +22,10 @@ public class ImageServiceResponse {

    @JsonProperty(value = "imageMetadata")
    @JsonAlias("data")
+    @Builder.Default
    private List<ImageMetadata> data = new ArrayList<>();

+    @Builder.Default
    private List<ImageMetadata> dataCV = new ArrayList<>();


@@ -15,6 +15,7 @@ import lombok.NoArgsConstructor;
 public class TableData {

    private PageInfo pageInfo;
+    @Builder.Default
    private List<TableCells> tableCells = new ArrayList<>();

 }
@@ -19,7 +19,7 @@ public class TableServiceResponse {
    private String operation;
    private String targetFileExtension;
    private String responseFileExtension;
-
+    @Builder.Default
    private List<TableData> data = new ArrayList<>();

 }
@@ -23,11 +23,11 @@ public class DividingColumnDetectionService {
    public List<Rectangle2D> detectColumns(PageContents pageContents) {


-        if (pageContents.getSortedTextPositionSequences().size() < 2) {
+        if (pageContents.getSortedWords().size() < 2) {
            return List.of(pageContents.getCropBox());
        }

-        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), pageContents.getCropBox());
+        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), pageContents.getCropBox());

        return detectColumnsFromLines(linesWithGapInformation.getXGaps(), pageContents.getCropBox());
    }
@@ -5,8 +5,7 @@ import java.util.LinkedList;
 import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.AllArgsConstructor;
 import lombok.experimental.UtilityClass;
@@ -19,26 +18,26 @@ public class GapDetectionService {
    private static final double NEW_LINE_FACTOR = 0.2;


-    public static GapInformation findGapsInLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
+    public static GapInformation findGapsInLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {

-        if (sortedTextPositionSequences.isEmpty()) {
+        if (sortedWords.isEmpty()) {
            return new GapInformation();
        }

-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);

        XGapsContext xGapContext = XGapsContext.init(mainBodyTextFrame);
        YGapsContext yGapContext = YGapsContext.init(mainBodyTextFrame);

-        var previousTextPosition = sortedTextPositionSequences.get(0);
+        var previousTextPosition = sortedWords.get(0);
        Rectangle2D rectangle = toRectangle2D(previousTextPosition);

        xGapContext.addGapFromLeftEdgeOfMainBody(rectangle);

-        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
+        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {

            double yDifference = Math.abs(currentTextPosition.getMaxYDirAdj() - previousTextPosition.getMaxYDirAdj());
-            double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getMinXDirAdj());
+            double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj());
            Rectangle2D previousTextPositionBBox = toRectangle2D(previousTextPosition);
            Rectangle2D currentTextPositionBBox = toRectangle2D(currentTextPosition);

@@ -60,14 +59,14 @@ public class GapDetectionService {
            }
            previousTextPosition = currentTextPosition;
        }
-        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedTextPositionSequences.get(sortedTextPositionSequences.size() - 1)));
+        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedWords.get(sortedWords.size() - 1)));
        xGapContext.gapsPerLine.add(xGapContext.gapsInCurrentLine);

        return new GapInformation(xGapContext.gapsPerLine, yGapContext.gapsPerLine);
    }


-    private static Rectangle2D toRectangle2D(TextPositionSequence textPosition) {
+    private static Rectangle2D toRectangle2D(Word textPosition) {

        return mirrorY(textPosition.getBBox());
    }
@@ -88,18 +87,18 @@ public class GapDetectionService {
    }


-    private static void assertAllTextPositionsHaveSameDir(List<TextPositionSequence> textPositionSequences) {
+    private static void assertAllTextPositionsHaveSameDir(List<Word> words) {

-        assert textPositionSequences.stream()
-                .map(TextPositionSequence::getDir)
-                .allMatch(a -> a.equals(textPositionSequences.get(0).getDir()));
+        assert words.stream()
+                .map(Word::getDir)
+                .allMatch(a -> a.equals(words.get(0).getDir()));
    }


-    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
+    private static double getAvgTextPositionHeight(List<Word> words) {

-        return textPositionSequences.stream()
-                .mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
+        return words.stream()
+                .mapToDouble(Word::getHeight).average().orElseThrow();
    }


@@ -7,17 +7,17 @@ import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@UtilityClass
 public class InvisibleTableDetectionService {

-    public List<List<Rectangle2D>> detectTable(List<TextPositionSequence> textPositionSequences, Rectangle2D tableBBox) {
+    public List<List<Rectangle2D>> detectTable(List<Word> words, Rectangle2D tableBBox) {

-        LineInformation lineInformation = LineDetectionService.calculateLineInformation(textPositionSequences);
-        GapInformation gaps = GapDetectionService.findGapsInLines(textPositionSequences, tableBBox);
+        LineInformation lineInformation = LineDetectionService.calculateLineInformation(words);
+        GapInformation gaps = GapDetectionService.findGapsInLines(words, tableBBox);
        List<Rectangle2D> gapsAcrossLines = GapsAcrossLinesService.detectXGapsAcrossLines(gaps, tableBBox);
        List<Double> columnXCoords = gapsAcrossLines.stream().map(RectangularShape::getCenterX).toList();
        int colCount = gapsAcrossLines.size();
@@ -7,7 +7,7 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -19,37 +19,37 @@ public class LineDetectionService {
    private static final double X_GAP_FACTOR = 1; // multiplied with average text height, determines the minimum distance of gaps in lines


-    public LineInformation calculateLineInformation(List<TextPositionSequence> sortedTextPositionSequences) {
+    public LineInformation calculateLineInformation(List<Word> sortedWords) {

-        if (sortedTextPositionSequences.isEmpty()) {
+        if (sortedWords.isEmpty()) {
            return LineFactory.init().build();
        }

-        return buildLineInformation(sortedTextPositionSequences);
+        return buildLineInformation(sortedWords);
    }


-    public List<List<Rectangle2D>> findLinesWithGaps(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
+    public List<List<Rectangle2D>> findLinesWithGaps(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {

-        return calculateLineInformation(sortedTextPositionSequences).getBBoxWithGapsByLines();
+        return calculateLineInformation(sortedWords).getBBoxWithGapsByLines();
    }


-    public List<List<TextPositionSequence>> orderByLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
+    public List<List<Word>> orderByLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {

-        return calculateLineInformation(sortedTextPositionSequences).getSequencesByLines();
+        return calculateLineInformation(sortedWords).getSequencesByLines();
    }


-    private static LineInformation buildLineInformation(List<TextPositionSequence> sortedTextPositionSequences) {
+    private static LineInformation buildLineInformation(List<Word> sortedWords) {

-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);

        LineFactory lineFactory = LineFactory.init();

-        var previousTextPosition = sortedTextPositionSequences.get(0);
+        var previousTextPosition = sortedWords.get(0);
        lineFactory.addToCurrentLine(previousTextPosition);
-        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
+        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {
            if (isNewLine(currentTextPosition, previousTextPosition, avgTextPositionHeight) || isSplitByOrientation(currentTextPosition, previousTextPosition)) {
                lineFactory.startNewLine();
            } else if (isXGap(currentTextPosition, previousTextPosition, avgTextPositionHeight)) {
@@ -63,27 +63,27 @@ public class LineDetectionService {
    }


-    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
+    private static double getAvgTextPositionHeight(List<Word> words) {

-        return textPositionSequences.stream().mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
+        return words.stream().mapToDouble(Word::getHeight).average().orElseThrow();
    }


-    private static boolean isXGap(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
+    private static boolean isXGap(Word currentTextPosition, Word previousTextPosition, double avgTextPositionHeight) {

-        return Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getMinXDirAdj()) > (avgTextPositionHeight * X_GAP_FACTOR);
+        return Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj()) > (avgTextPositionHeight * X_GAP_FACTOR);
    }


-    private static boolean isSplitByOrientation(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition) {
+    private static boolean isSplitByOrientation(Word currentTextPosition, Word previousTextPosition) {

        return !previousTextPosition.getDir().equals(currentTextPosition.getDir());
    }


-    private static boolean isNewLine(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
+    private static boolean isNewLine(Word currentTextPosition, Word previousTextPosition, double avgTextPositionHeight) {

-        return Math.abs(previousTextPosition.getMinYDirAdj() - currentTextPosition.getMinYDirAdj()) > avgTextPositionHeight;
+        return Math.abs(previousTextPosition.getYDirAdj() - currentTextPosition.getYDirAdj()) > avgTextPositionHeight;
    }


@@ -96,13 +96,13 @@ public class LineDetectionService {
        List<List<Rectangle2D>> bBoxWithGapsByLines;
        List<Rectangle2D> bBoxWithGapsInCurrentLine;

-        List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
-        List<List<TextPositionSequence>> sequencesWithGapsInCurrentLine;
+        List<List<List<Word>>> sequencesWithGapsByLines;
+        List<List<Word>> sequencesWithGapsInCurrentLine;

-        List<TextPositionSequence> currentSequencesWithoutGaps;
+        List<Word> currentSequencesWithoutGaps;

-        List<List<TextPositionSequence>> sequencesByLines;
-        List<TextPositionSequence> sequencesInCurrentLine;
+        List<List<Word>> sequencesByLines;
+        List<Word> sequencesInCurrentLine;

        List<List<Rectangle2D>> xGaps;
        List<List<Rectangle2D>> yGaps;
@@ -116,14 +116,14 @@ public class LineDetectionService {
            List<Rectangle2D> bBoxWithGapsInCurrentLine = new LinkedList<>();
            bBoxWithGapsByLines.add(bBoxWithGapsInCurrentLine);

-            List<List<List<TextPositionSequence>>> sequencesWithGapsByLines = new LinkedList<>();
-            List<List<TextPositionSequence>> sequencesWithGapsInCurrentLine = new LinkedList<>();
+            List<List<List<Word>>> sequencesWithGapsByLines = new LinkedList<>();
+            List<List<Word>> sequencesWithGapsInCurrentLine = new LinkedList<>();
            sequencesWithGapsByLines.add(sequencesWithGapsInCurrentLine);
-            List<TextPositionSequence> currentSequencesWithoutGaps = new LinkedList<>();
+            List<Word> currentSequencesWithoutGaps = new LinkedList<>();
            sequencesWithGapsInCurrentLine.add(currentSequencesWithoutGaps);

-            List<List<TextPositionSequence>> sequencesByLines = new LinkedList<>();
-            List<TextPositionSequence> sequencesInCurrentLine = new LinkedList<>();
+            List<List<Word>> sequencesByLines = new LinkedList<>();
+            List<Word> sequencesInCurrentLine = new LinkedList<>();
            sequencesByLines.add(sequencesInCurrentLine);

            return new LineFactory(lineBBox,
@@ -178,13 +178,13 @@ public class LineDetectionService {
        }


-        private Rectangle2D textPositionBBox(List<TextPositionSequence> textPositionSequences) {
+        private Rectangle2D textPositionBBox(List<Word> words) {

-            return RectangleTransformations.rectangle2DBBox(textPositionSequences.stream().map(TextPositionSequence::getBBox).toList());
+            return RectangleTransformations.rectangle2DBBox(words.stream().map(Word::getBBox).toList());
        }


-        public void addToCurrentLine(TextPositionSequence current) {
+        public void addToCurrentLine(Word current) {

            sequencesInCurrentLine.add(current);
            currentSequencesWithoutGaps.add(current);
@@ -13,7 +13,7 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.springframework.core.io.ClassPathResource;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

@@ -40,7 +40,7 @@ public class PageContentExtractor {
                stripper.setPdpage(pdPage);
                stripper.getText(pdDocument);

-                Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
+                Map<Float, List<Word>> sortedTextPositionSequencesPerDir = stripper.getWords()
                        .stream()
                        .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));

@@ -57,7 +57,7 @@ public class PageContentExtractor {
    }


-    public List<TextPositionSequence> sortByDirAccordingToPageRotation(Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir, int rotation) {
+    public List<Word> sortByDirAccordingToPageRotation(Map<Float, List<Word>> sortedTextPositionSequencesPerDir, int rotation) {

        LinkedList<Float> sortedKeys = new LinkedList<>(sortedTextPositionSequencesPerDir.keySet().stream().sorted().toList());

@@ -14,9 +14,9 @@ public class PageInformationService {

    public PageInformation build(PageContents pageContents) {

-        LineInformation lineInformation = LineDetectionService.calculateLineInformation(pageContents.getSortedTextPositionSequences());
+        LineInformation lineInformation = LineDetectionService.calculateLineInformation(pageContents.getSortedWords());
        Rectangle2D mainBodyTextFrame = MainBodyTextFrameExtractionService.calculateMainBodyTextFrame(lineInformation);
-        GapInformation gapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), mainBodyTextFrame);
+        GapInformation gapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), mainBodyTextFrame);

        return new PageInformation(pageContents, lineInformation, mainBodyTextFrame, gapInformation);
    }
@@ -78,7 +78,7 @@ public class TableExtractionService {

            List<Cell> containedCells = new ArrayList<>();
            for (Cell c : cells) {
-                if (c.hasMinimumSize() && area.contains(c.getBBoxInitialUserSpace())) {
+                if (c.hasMinimumSize() && area.contains(c.getBBoxPdf())) {
                    containedCells.add(c);
                }
            }
@@ -5,7 +5,7 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@@ -17,9 +17,9 @@ public class TextRulingsClassifier {
    private final static double TEXT_BBOX_THRESHOLD_FACTOR = 0.15; // multiplied with text width then subtracted from word width. If ruling covers this width, it is considered as strikethrough/underline.


-    public static void classifyUnderlinedAndStrikethroughText(List<TextPositionSequence> words, CleanRulings cleanRulings) {
+    public static void classifyUnderlinedAndStrikethroughText(List<Word> words, CleanRulings cleanRulings) {

-        for (TextPositionSequence word : words) {
+        for (Word word : words) {
            if (word.getDir().equals(TextDirection.ZERO) || word.getDir().equals(TextDirection.HALF_CIRCLE)) {
                handleHorizontalText(cleanRulings, word);
            } else {
@@ -29,15 +29,15 @@ public class TextRulingsClassifier {
    }


-    private static void handleVerticalText(CleanRulings cleanRulings, TextPositionSequence word) {
+    private static void handleVerticalText(CleanRulings cleanRulings, Word word) {

-        float lowerY = (float) (word.getBBoxInitialUserSpace().getMinY() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
-        float upperY = (float) (word.getBBoxInitialUserSpace().getMaxY() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
+        float lowerY = (float) (word.getBBoxPdf().getMinY() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
+        float upperY = (float) (word.getBBoxPdf().getMaxY() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());

-        float strikethroughCenterX = (float) word.getBBoxInitialUserSpace().getCenterX();
+        float strikethroughCenterX = (float) word.getBBoxPdf().getCenterX();
        float strikethroughBoxHeight = (float) ((word.getBBoxDirAdj().getHeight() * STRIKETHROUGH_ZONE) / 2);

-        float underlineCenterX = (float) (word.getDir().equals(TextDirection.QUARTER_CIRCLE) ? word.getBBoxInitialUserSpace().getMaxX() : word.getBBoxInitialUserSpace().getMinX());
+        float underlineCenterX = (float) (word.getDir().equals(TextDirection.QUARTER_CIRCLE) ? word.getBBoxPdf().getMaxX() : word.getBBoxPdf().getMinX());
        float underlineBoxHeight = (float) ((word.getBBoxDirAdj().getHeight() * UNDERLINE_ZONE) / 2);

        float leftX = Math.min(underlineCenterX - underlineBoxHeight, strikethroughCenterX - strikethroughBoxHeight);
@@ -63,15 +63,15 @@ public class TextRulingsClassifier {
    }


-    private static void handleHorizontalText(CleanRulings cleanRulings, TextPositionSequence word) {
+    private static void handleHorizontalText(CleanRulings cleanRulings, Word word) {

-        float leftX = (float) (word.getBBoxInitialUserSpace().getMinX() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
-        float rightX = (float) (word.getBBoxInitialUserSpace().getMaxX() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
+        float leftX = (float) (word.getBBoxPdf().getMinX() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
+        float rightX = (float) (word.getBBoxPdf().getMaxX() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());

-        float strikethroughCenterY = (float) word.getBBoxInitialUserSpace().getCenterY();
+        float strikethroughCenterY = (float) word.getBBoxPdf().getCenterY();
        float strikethroughBoxHeight = (float) ((word.getBBoxDirAdj().getHeight() * STRIKETHROUGH_ZONE) / 2);

-        float underlineCenterY = (float) (word.getDir().equals(TextDirection.ZERO) ? word.getBBoxInitialUserSpace().getMinY() : word.getBBoxInitialUserSpace().getMaxY());
+        float underlineCenterY = (float) (word.getDir().equals(TextDirection.ZERO) ? word.getBBoxPdf().getMinY() : word.getBBoxPdf().getMaxY());
        float underlineBoxHeight = (float) ((word.getBBoxDirAdj().getHeight() * UNDERLINE_ZONE) / 2);

        float lowerY = Math.min(underlineCenterY - underlineBoxHeight, strikethroughCenterY - strikethroughBoxHeight);
@@ -2,14 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor.services.blockifica

 import static com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService.buildTextBlock;

-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Locale;
-import java.util.function.Function;

-import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
@@ -18,24 +16,15 @@ import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.Data;

@Service
 public class BlockificationPostprocessingService {

-    private static final float BLOCK_TO_OUTLINE_DISTANCE_THRESHOLD = 5.0f;
-
-    private static final Function<TextPageBlock, Rectangle2D> blockToBoundingBox = (abstractPageBlock) -> abstractPageBlock.getSequences()
-            .stream()
-            .map(textPositionSequence -> textPositionSequence.getTextPositions()
-                    .stream()
-                    .map(tp -> SearchTextWithTextPositionFactory.mapRedTextPositionToInitialUserSpace(tp, textPositionSequence))
-                    .collect(RectangleTransformations.collectBBox()))
-            .collect(RectangleTransformations.collectBBox());
+    private static final float STRING_SIMILARITY_THRESHOLD = 0.1f;


    public OutlineObject sanitizeOutlineBlocks(ClassificationPage classificationPage, OutlineObject notFoundOutlineObject) {
@@ -46,20 +35,18 @@ public class BlockificationPostprocessingService {
            return null;
        }

-        float pageHeight = classificationPage.getPageHeight();
-
        ListIterator<OutlineObject> outlineObjectListIterator = outlineObjects.listIterator();

        if (notFoundOutlineObject != null) {
            OutlineProcessionContext notFoundOutlineObjectProcessionContext = new OutlineProcessionContext(notFoundOutlineObject);
-            processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, notFoundOutlineObjectProcessionContext);
+            processTextBlocks(getTextPageBlocks(classificationPage), notFoundOutlineObjectProcessionContext);

            OutlineObject firstOutlineObject = null;
            OutlineProcessionContext firstOutlineObjectProcessionContext = null;
            if (outlineObjectListIterator.hasNext()) {
                firstOutlineObject = outlineObjectListIterator.next();
                firstOutlineObjectProcessionContext = new OutlineProcessionContext(firstOutlineObject);
-                processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, firstOutlineObjectProcessionContext);
+                processTextBlocks(getTextPageBlocks(classificationPage), firstOutlineObjectProcessionContext);
            }

            if (!contextsOverlap(notFoundOutlineObjectProcessionContext, firstOutlineObjectProcessionContext)) {
@@ -68,7 +55,7 @@ public class BlockificationPostprocessingService {
            if (firstOutlineObject != null) {
                // re-create the context for the updated blocks
                firstOutlineObjectProcessionContext = new OutlineProcessionContext(firstOutlineObject);
-                processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, firstOutlineObjectProcessionContext);
+                processTextBlocks(getTextPageBlocks(classificationPage), firstOutlineObjectProcessionContext);
                firstOutlineObject.setFound(selectMatch(classificationPage, firstOutlineObjectProcessionContext));
            }

@@ -76,7 +63,7 @@ public class BlockificationPostprocessingService {

        outlineObjectListIterator.forEachRemaining(outlineObject -> {
            OutlineProcessionContext outlineObjectProcessionContext = new OutlineProcessionContext(outlineObject);
-            processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, outlineObjectProcessionContext);
+            processTextBlocks(getTextPageBlocks(classificationPage), outlineObjectProcessionContext);
            outlineObject.setFound(selectMatch(classificationPage, outlineObjectProcessionContext));
        });

@@ -116,8 +103,7 @@ public class BlockificationPostprocessingService {

        double maxYFirst = blocksOfFirstOutline.stream()
                .mapToDouble(TextPageBlock::getPdfMaxY)
-                .max()
-                .orElse(Double.NEGATIVE_INFINITY);
+                .max().orElse(Double.NEGATIVE_INFINITY);

        return blocksOfNotFoundOutline.stream()
                .mapToDouble(TextPageBlock::getPdfMaxY)
@@ -139,13 +125,13 @@ public class BlockificationPostprocessingService {
    }


-    private void processTextBlocks(List<TextPageBlock> textBlocks, float pageHeight, OutlineProcessionContext context) {
+    private void processTextBlocks(List<TextPageBlock> textBlocks, OutlineProcessionContext context) {

        OutlineObject outlineObject = context.getOutlineObject();
        ListIterator<TextPageBlock> iterator = textBlocks.listIterator();
        while (iterator.hasNext()) {
            TextPageBlock pageBlock = iterator.next();
-            if (pageHeight - outlineObject.getPoint().getY() - BLOCK_TO_OUTLINE_DISTANCE_THRESHOLD <= pageBlock.getMaxY()) {
+            if (outlineObject.isAbove(pageBlock)) {
                break;
            }
        }
@@ -190,8 +176,7 @@ public class BlockificationPostprocessingService {
            for (List<TextPageBlock> combination : combinations) {
                double averageDistance = combination.stream()
                        .map(block -> calculateDistance(outlineObject, block))
-                        .mapToDouble(Double::doubleValue).average()
-                        .orElse(Double.MAX_VALUE);
+                        .mapToDouble(Double::doubleValue).average().orElse(Double.MAX_VALUE);
                if (distanceToBestMergeCandidates > averageDistance) {
                    distanceToBestMergeCandidates = averageDistance;
                    bestMergeCandidateCombination = combination;
@@ -237,14 +222,14 @@ public class BlockificationPostprocessingService {
            headline = sectionIdentifier + headline;
        }

-        WordSequenceResult wordSequenceResult = findWordSequence(blockToSplit.getSequences(), headline);
+        WordSequenceResult wordSequenceResult = findWordSequence(blockToSplit.getWords(), headline);
        if (wordSequenceResult.inSequence.isEmpty() && !headline.equals(title)) {
-            wordSequenceResult = findWordSequence(blockToSplit.getSequences(), title);
+            wordSequenceResult = findWordSequence(blockToSplit.getWords(), title);
        }

        boolean modifiedBlockToSplit = false;
        if (!wordSequenceResult.inSequence.isEmpty()) {
-            blockToSplit.setSequences(wordSequenceResult.inSequence);
+            blockToSplit.setWords(wordSequenceResult.inSequence);
            blockToSplit.recalculateBBox();
            modifiedBlockToSplit = true;
        }
@@ -265,15 +250,19 @@ public class BlockificationPostprocessingService {
    }


-    private static WordSequenceResult findWordSequence(List<TextPositionSequence> textPositionSequences, String text) {
+    private static WordSequenceResult findWordSequence(List<Word> words, String text) {

        String target = sanitizeString(text);
-        List<TextPositionSequence> inSequence = new ArrayList<>();
-        List<TextPositionSequence> preSequence = new ArrayList<>();
-        List<TextPositionSequence> postSequence = new ArrayList<>();
+        List<Word> inSequence = new ArrayList<>();
+        List<Word> preSequence = new ArrayList<>();
+        List<Word> postSequence = new ArrayList<>();
        StringBuilder currentSequence = new StringBuilder();

-        for (TextPositionSequence sequence : textPositionSequences) {
+        if (target.isBlank()) {
+            return new WordSequenceResult();
+        }
+
+        for (Word sequence : words) {

            currentSequence.append(sanitizeString(sequence.toString()));
            inSequence.add(sequence);
@@ -285,10 +274,10 @@ public class BlockificationPostprocessingService {
                    int index = 0;
                    String toRemove = currentSequence.substring(0, currentSequence.length() - target.length());

-                    TextPositionSequence next = inSequence.get(index);
+                    Word next = inSequence.get(index);
                    while (currentSequence.length() - next.length() >= target.length()) {

-                        TextPositionSequence removed = inSequence.remove(index);
+                        Word removed = inSequence.remove(index);
                        currentSequence.delete(0, removed.toString().length());
                        preSequence.add(removed);

@@ -317,7 +306,7 @@ public class BlockificationPostprocessingService {
                }

                if (currentSequence.toString().equals(target)) {
-                    postSequence.addAll(textPositionSequences.subList(textPositionSequences.indexOf(sequence) + 1, textPositionSequences.size()));
+                    postSequence.addAll(words.subList(words.indexOf(sequence) + 1, words.size()));
                    return new WordSequenceResult(inSequence, preSequence, postSequence);
                }
            }
@@ -327,12 +316,12 @@ public class BlockificationPostprocessingService {
    }


-    private static SplitSequenceResult splitSequence(TextPositionSequence sequence, String toRemove) {
+    private static SplitSequenceResult splitSequence(Word sequence, String toRemove) {

-        TextPositionSequence in = null;
-        TextPositionSequence out;
+        Word in = null;
+        Word out;

-        String currentSequence = sequence.toString();
+        String currentSequence = sequence.toString().toLowerCase(Locale.ROOT);
        int index = currentSequence.indexOf(toRemove);
        int endIndex = index + toRemove.length();

@@ -348,9 +337,9 @@ public class BlockificationPostprocessingService {
    }


-    private static TextPositionSequence createSubSequence(TextPositionSequence sequence, int start, int end) {
+    private static Word createSubSequence(Word sequence, int start, int end) {

-        TextPositionSequence newSeq = new TextPositionSequence(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
+        Word newSeq = new Word(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
        newSeq.setParagraphStart(sequence.isParagraphStart());
        return newSeq;
    }
@@ -365,10 +354,10 @@ public class BlockificationPostprocessingService {
            List<TextPageBlock> mergedBlocks = new ArrayList<>();
            for (TextPageBlock textPageBlock : blocksToMerge.subList(1, blocksToMerge.size())) {

-                if (firstBlock != null && !firstBlock.getSequences().isEmpty()) {
+                if (firstBlock != null && !firstBlock.getWords().isEmpty()) {

                    if (textPageBlock.getDir() == firstBlock.getDir()) {
-                        firstBlock.getSequences().addAll(textPageBlock.getSequences());
+                        firstBlock.addAll(textPageBlock.getWords());
                        mergedBlocks.add(textPageBlock);
                    }
                }
@@ -416,9 +405,7 @@ public class BlockificationPostprocessingService {

    private double calculateDistance(OutlineObject outlineObject, TextPageBlock pageBlock) {

-        double deltaX = outlineObject.getPoint().getX() - pageBlock.getMinX();
-        double deltaY = pageBlock.getPageHeight() - outlineObject.getPoint().getY() - pageBlock.getMinY();
-        return Math.sqrt(deltaX * deltaX + deltaY * deltaY);
+        return outlineObject.distance(pageBlock);
    }


@@ -435,6 +422,13 @@ public class BlockificationPostprocessingService {
        String blockText = sanitizeString(pageBlock.getText());
        String outlineTitle = sanitizeString(outlineObject.getTitle());

+        int threshold = (int) (Math.min(blockText.length(), outlineTitle.length()) * STRING_SIMILARITY_THRESHOLD) + 1;
+        int distance = new LevenshteinDistance(threshold).apply(blockText, outlineTitle);
+        if (distance >= 0 && distance < threshold) {
+            context.directMatch = pageBlock;
+            return true;
+        }
+
        boolean blockTextContainsOutlineTitle = blockText.contains(outlineTitle);
        boolean outlineTitleContainsBlockText = outlineTitle.contains(blockText);

@@ -473,7 +467,9 @@ public class BlockificationPostprocessingService {

    private static String sanitizeString(String text) {

-        return StringUtils.deleteWhitespace(text).toLowerCase(Locale.ROOT);
+        return TextNormalizationUtilities.removeAllWhitespaces(text)//
+                .trim() // sometimes there are trailing empty bytes at the end of the string trim() seems to remove them
+                .toLowerCase(Locale.ENGLISH);
    }


@@ -500,12 +496,12 @@ public class BlockificationPostprocessingService {

    public static class WordSequenceResult {

-        public List<TextPositionSequence> inSequence;
-        public List<TextPositionSequence> preSequence;
-        public List<TextPositionSequence> postSequence;
+        public List<Word> inSequence;
+        public List<Word> preSequence;
+        public List<Word> postSequence;


-        public WordSequenceResult(List<TextPositionSequence> inSequence, List<TextPositionSequence> preSequence, List<TextPositionSequence> postSequence) {
+        public WordSequenceResult(List<Word> inSequence, List<Word> preSequence, List<Word> postSequence) {

            this.inSequence = inSequence;
            this.preSequence = preSequence;
@@ -526,7 +522,7 @@ public class BlockificationPostprocessingService {

    }

-    public record SplitSequenceResult(TextPositionSequence in, TextPositionSequence out) {
+    public record SplitSequenceResult(Word in, Word out) {

    }

@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;

 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
 import java.util.ListIterator;

@@ -10,13 +9,12 @@ import org.springframework.stereotype.Service;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;

 import lombok.RequiredArgsConstructor;
@@ -32,7 +30,7 @@ public class DocstrumBlockificationService {
    static final float THRESHOLD = 1f;


-    public ClassificationPage blockify(List<TextPositionSequence> textPositions,
+    public ClassificationPage blockify(List<Word> textPositions,
                                       CleanRulings rulings,
                                       boolean xyOrder,
                                       LayoutDebugLayer visualizations,
@@ -40,7 +38,7 @@ public class DocstrumBlockificationService {

        CleanRulings usedRulings = rulings.withoutTextRulings();

-        var zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder, usedRulings, visualizations);
+        List<Zone> zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder, usedRulings, visualizations);

        if (!textPositions.isEmpty()) {
            visualizations.addZoneVisualizations(zones, textPositions.get(0).getPage());
@@ -48,11 +46,7 @@ public class DocstrumBlockificationService {
            visualizations.addCharactersWithNeighbours(zones, textPositions.get(0).getPage());
        }

-        var pageBlocks = toAbstractPageBlocks(zones, xyOrder, usedRulings);
-
-        if (xyOrder) {
-            sortPageBlocksXThenY(pageBlocks);
-        }
+        var pageBlocks = toAbstractPageBlocks(zones);

        var classificationPage = new ClassificationPage(pageBlocks);
        classificationPage.setCleanRulings(rulings);
@@ -73,35 +67,21 @@ public class DocstrumBlockificationService {
    }


-    private static void sortPageBlocksXThenY(List<AbstractPageBlock> pageBlocks) {
-
-        pageBlocks.sort(Comparator.comparing(AbstractPageBlock::getMinY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                                .thenComparing(AbstractPageBlock::getMinX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-        pageBlocks.sort(new Comparator<AbstractPageBlock>() {
-            @Override
-            public int compare(AbstractPageBlock o1, AbstractPageBlock o2) {
-
-                return Math.abs(o1.getMinY() - o2.getMinY()) < 5 && o1.getMinX() < o2.getMinX() == true ? -1 : 0;
-            }
-        });
-    }
-
-
-    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones, boolean xyOrder, CleanRulings usedRulings) {
+    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones) {

        List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
        zones.forEach(zone -> {

-            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+            List<Word> words = new ArrayList<>();
            zone.getLines()
                    .forEach(line -> {
                        line.getWords()
                                .forEach(word -> {
-                                    textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), word.getPage()));
+                                    words.add(new Word(word.getTextPositions(), word.getPage()));
                                });
                    });

-            abstractPageBlocks.add(buildTextBlock(textPositionSequences, 0));
+            abstractPageBlocks.add(buildTextBlock(words, 0));
        });

        return abstractPageBlocks;
@@ -122,7 +102,7 @@ public class DocstrumBlockificationService {
            }
            TextPageBlock current = (TextPageBlock) block;

-            if (previous != null && !previous.getSequences().isEmpty()) {
+            if (previous != null && !previous.getWords().isEmpty()) {

                if (current.getDir() != previous.getDir() || usedRulings.lineBetween(current, previous)) {
                    previous = current;
@@ -202,8 +182,8 @@ public class DocstrumBlockificationService {

    private TextPageBlock combineBlocksAndResetIterator(TextPageBlock previous, TextPageBlock current, ListIterator<AbstractPageBlock> itty, boolean toDuplicate) {

-        previous.getSequences().addAll(current.getSequences());
-        previous = buildTextBlock(previous.getSequences(), 0);
+        previous.addAll(current.getWords());
+        previous = buildTextBlock(previous.getWords(), 0);
        previous.setToDuplicate(toDuplicate);
        if (current.getClassification() != null && previous.getClassification() == null) {
            previous.setClassification(current.getClassification());
@@ -296,11 +276,15 @@ public class DocstrumBlockificationService {
                    continue;
                }

+//                if (!current.getMostPopularWordStyle().equals(inner.getMostPopularWordStyle())) {
+//                    continue;
+//                }
+
                if (current.getDir() == inner.getDir() && current.intersects(inner, yThreshold, xThreshold)) {

                    boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
-                    current.getSequences().addAll(inner.getSequences());
-                    current = buildTextBlock(current.getSequences(), 0);
+                    current.addAll(inner.getWords());
+                    current = buildTextBlock(current.getWords(), 0);

                    current.setToDuplicate(toDuplicate);
                    blocks.set(i, null);
@@ -317,7 +301,7 @@ public class DocstrumBlockificationService {
    }


-    public static TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    public static TextPageBlock buildTextBlock(List<Word> wordBlockList, int indexOnPage) {

        return new TextPageBlock(wordBlockList);
    }
@@ -1,29 +1,29 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;

-import static java.util.stream.Collectors.toSet;
-
 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

+@SuppressWarnings("all")
@Service
 public class DocuMineBlockificationService {

    static final float THRESHOLD = 1f;
+    public static final double FONT_SIZE_CHANGE_RATIO = 0.15;

    Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);

@@ -37,38 +37,35 @@ public class DocuMineBlockificationService {
     * @param cleanRulings  All rulings on a page
     * @return Page object that contains the Textblock and text statistics.
     */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, CleanRulings cleanRulings) {
+    public ClassificationPage blockify(List<Word> textPositions, CleanRulings cleanRulings) {

-        List<TextPositionSequence> chunkWords = new ArrayList<>();
+        List<Word> chunkWords = new ArrayList<>();
        List<AbstractPageBlock> textPageBlocks = new ArrayList<>();

        CleanRulings usedRulings = cleanRulings.withoutTextRulings();

-        float minX = 1000;
-        float maxX = 0;
-        float minY = 1000;
-        float maxY = 0;
-        TextPositionSequence prev = null;
+        double minX = 1000;
+        double maxX = 0;
+        double minY = 1000;
+        double maxY = 0;
+        Word prev = null;

        boolean wasSplitted = false;
-        Float splitX1 = null;
-        for (TextPositionSequence word : textPositions) {
+        Double splitX1 = null;
+        for (Word word : textPositions) {

-            boolean lineSeparation = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * 1.25;
-            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
-            boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
-            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < -5;
-            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
+            boolean lineSeparation = prev != null && word.getYDirAdj() - prev.getMaxYDirAdj() > Math.min(word.getHeight(), prev.getHeight()) * 1.1;
+            boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
+            boolean splitByX = prev != null && maxX + 50 < word.getXDirAdj() && prev.getYDirAdj() == word.getYDirAdj();
+            boolean negativeXGap = prev != null && word.getXDirAdj() - minX < -5;
+            boolean newLineAfterSplit = prev != null && word.getYDirAdj() != prev.getYDirAdj() && wasSplitted && splitX1 != word.getXDirAdj();
            boolean isSplitByRuling = prev != null && usedRulings.lineBetween(prev, word);
            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 && (word.getFontStyle()
-                                                                                                                                                               .contains("bold")
-                                                                                                                                                       && !prev.getFontStyle()
-                    .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
+            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 && isFontChange(word, prev);

            Matcher matcher = pattern.matcher(chunkWords.stream()
                                                      .collect(Collectors.joining(" ")).toString());
-            boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();
+            boolean startsOnSameX = Math.abs(minX - word.getXDirAdj()) < 5 && matcher.matches();

            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap || startsOnSameX)) {

@@ -84,7 +81,7 @@ public class DocuMineBlockificationService {
                if (splitByX && !isSplitByRuling) {
                    wasSplitted = true;
                    cb1.setOrientation(Orientation.LEFT);
-                    splitX1 = word.getMinXDirAdj();
+                    splitX1 = word.getXDirAdj();
                } else if (newLineAfterSplit && !isSplitByRuling) {
                    wasSplitted = false;
                    cb1.setOrientation(Orientation.RIGHT);
@@ -107,14 +104,14 @@ public class DocuMineBlockificationService {
            chunkWords.add(word);

            prev = word;
-            if (word.getMinXDirAdj() < minX) {
-                minX = word.getMinXDirAdj();
+            if (word.getXDirAdj() < minX) {
+                minX = word.getXDirAdj();
            }
            if (word.getMaxXDirAdj() > maxX) {
                maxX = word.getMaxXDirAdj();
            }
-            if (word.getMinYDirAdj() < minY) {
-                minY = word.getMinYDirAdj();
+            if (word.getYDirAdj() < minY) {
+                minY = word.getYDirAdj();
            }
            if (word.getMaxYDirAdj() > maxY) {
                maxY = word.getMaxYDirAdj();
@@ -127,6 +124,85 @@ public class DocuMineBlockificationService {
    }


+    private static boolean isFontChange(Word word, Word prev) {
+
+        return word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold")
+               || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold")
+               || Math.abs(prev.getFontSize() - word.getFontSize()) >= FONT_SIZE_CHANGE_RATIO * Math.min(prev.getFontSize(), word.getFontSize())
+               || Math.abs(word.getTextHeight() - prev.getTextHeight()) >= FONT_SIZE_CHANGE_RATIO * Math.min(prev.getTextHeight(), word.getTextHeight());
+    }
+
+
+    public void mergeblocks(ClassificationPage page, CleanRulings usedRulings, float xThreshold, float yThreshold) {
+
+        var blocks = page.getTextBlocks();
+        ListIterator<AbstractPageBlock> itty = blocks.listIterator();
+        while (itty.hasNext()) {
+            AbstractPageBlock block = itty.next();
+            if (block == null) {
+                continue;
+            }
+            if (block instanceof TablePageBlock) {
+                continue;
+            }
+
+            TextPageBlock current = (TextPageBlock) block;
+
+            for (int i = 0; i < blocks.size(); i++) {
+
+                AbstractPageBlock abstractPageBlock = blocks.get(i);
+                if (abstractPageBlock == null) {
+                    continue;
+                }
+                if (abstractPageBlock == current) {
+                    continue;
+                }
+                if (abstractPageBlock instanceof TablePageBlock) {
+                    continue;
+                }
+
+                if (isHeadlineFromOutline(current) || isHeadlineFromOutline(abstractPageBlock)) {
+                    continue;
+                }
+
+                TextPageBlock inner = (TextPageBlock) abstractPageBlock;
+
+                if (usedRulings.lineBetween(current, blocks.get(i))) {
+                    continue;
+                }
+
+                if (current.getDir() == inner.getDir() && current.intersects(inner, yThreshold, xThreshold) && (current.getClassification() == null || current.getClassification()
+                        .equals(inner.getClassification()))) {
+
+                    boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
+                    current.addAll(inner.getWords());
+                    current = buildTextBlock(current.getWords(), 0);
+                    current.setClassification(inner.getClassification());
+                    current.setToDuplicate(toDuplicate);
+                    blocks.set(i, null);
+                    itty.set(current);
+                }
+            }
+        }
+        var blocksIterator = blocks.iterator();
+        while (blocksIterator.hasNext()) {
+            if (blocksIterator.next() == null) {
+                blocksIterator.remove();
+            }
+        }
+    }
+
+
+    private boolean isHeadlineFromOutline(AbstractPageBlock abstractPageBlock) {
+
+        return abstractPageBlock.getEngines().contains(LayoutEngine.OUTLINE) && abstractPageBlock.getClassification() != null && abstractPageBlock.getClassification().isHeadline();
+    }
+
+
+    public static TextPageBlock buildTextBlock(List<Word> wordBlockList, int indexOnPage) {
+
+        return new TextPageBlock(wordBlockList);
+    }

 }

@@ -11,7 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;

@SuppressWarnings("all")
@@ -30,26 +30,26 @@ public class RedactManagerBlockificationService {
     * @param visualizations
     * @return Page object that contains the Textblock and text statistics.
     */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, CleanRulings cleanRulings, LayoutDebugLayer visualizations) {
+    public ClassificationPage blockify(List<Word> textPositions, CleanRulings cleanRulings, LayoutDebugLayer visualizations) {

        CleanRulings usedRulings = cleanRulings.withoutTextRulings();

        int indexOnPage = 0;
-        List<TextPositionSequence> chunkWords = new ArrayList<>();
+        List<Word> chunkWords = new ArrayList<>();
        List<AbstractPageBlock> chunkBlockList = new ArrayList<>();

-        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
-        TextPositionSequence prev = null;
+        double minX = 1000, maxX = 0, minY = 1000, maxY = 0;
+        Word prev = null;

        boolean wasSplitted = false;
-        Float splitX1 = null;
-        for (TextPositionSequence word : textPositions) {
+        Double splitX1 = null;
+        for (Word word : textPositions) {

-            boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25;
-            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
-            boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
-            boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
-            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
+            boolean lineSeparation = word.getYDirAdj() - maxY > word.getHeight() * 1.25;
+            boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
+            boolean splitByX = prev != null && maxX + 50 < word.getXDirAdj() && prev.getYDirAdj() == word.getYDirAdj();
+            boolean xIsBeforeFirstX = prev != null && word.getXDirAdj() < minX;
+            boolean newLineAfterSplit = prev != null && word.getYDirAdj() != prev.getYDirAdj() && wasSplitted && splitX1 != word.getXDirAdj();
            boolean isSplitByRuling = prev != null && usedRulings.lineBetween(prev, word);
            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());

@@ -69,7 +69,7 @@ public class RedactManagerBlockificationService {
                if (splitByX && !isSplitByRuling) {
                    wasSplitted = true;
                    cb1.setOrientation(Orientation.LEFT);
-                    splitX1 = word.getMinXDirAdj();
+                    splitX1 = word.getXDirAdj();
                } else if (newLineAfterSplit && !isSplitByRuling) {
                    wasSplitted = false;
                    cb1.setOrientation(Orientation.RIGHT);
@@ -92,14 +92,14 @@ public class RedactManagerBlockificationService {
            chunkWords.add(word);

            prev = word;
-            if (word.getMinXDirAdj() < minX) {
-                minX = word.getMinXDirAdj();
+            if (word.getXDirAdj() < minX) {
+                minX = word.getXDirAdj();
            }
            if (word.getMaxXDirAdj() > maxX) {
                maxX = word.getMaxXDirAdj();
            }
-            if (word.getMinYDirAdj() < minY) {
-                minY = word.getMinYDirAdj();
+            if (word.getYDirAdj() < minY) {
+                minY = word.getYDirAdj();
            }
            if (word.getMaxYDirAdj() > maxY) {
                maxY = word.getMaxYDirAdj();
@@ -161,7 +161,6 @@ public class RedactManagerBlockificationService {
        }
        if (!textPositions.isEmpty()) {
            visualizations.addTextBlockVisualizations(chunkBlockList.stream()
-                                                              .map(tb -> (TextPageBlock) tb)
                                                              .toList(), textPositions.get(0).getPage());
        }

@@ -10,7 +10,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -23,7 +22,7 @@ public class ClarifyndClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

@@ -35,7 +34,10 @@ public class ClarifyndClassificationService {
    }


-    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService,
+                              ClassificationPage page,
+                              ClassificationDocument document,
+                              List<Double> headlineFontSizes) {

        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextPageBlock) {
@@ -45,7 +47,11 @@ public class ClarifyndClassificationService {
    }


-    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
+                               TextPageBlock textBlock,
+                               ClassificationPage page,
+                               ClassificationDocument document,
+                               List<Double> headlineFontSizes) {

        var bodyTextFrame = page.getBodyTextFrame();

@@ -53,63 +59,58 @@ public class ClarifyndClassificationService {
            headlineClassificationService.setLastHeadlineFromOutline(textBlock);
            return;
        }
+        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
+                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
+                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
+            return;
+        }
        if (document.getFontSizeCounter().getMostPopular() == null) {
            textBlock.setClassification(PageBlockType.PARAGRAPH);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
-                textBlock,
-                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
-
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
-                textBlock,
-                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
-        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
-                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
-                .size() == 1)) {
+        if (page.getPageNumber() == 1 //
+            && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
+                && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification(PageBlockType.TITLE);
+                PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+                headlineClassificationService.classifyHeadline(textBlock, headlineType);
+                document.setHeadlines(true);
            }
-        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                .getCountPerValue()
-                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
+                   && PositionUtils.getApproxLineCount(textBlock) < 4.9
+                   && (textBlock.getMostPopularWordStyle().equals("bold")
+                       || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
+                          && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {

-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                    PageBlockType headlineType = PageBlockType.getHeadlineType(i);
-                    headlineClassificationService.classifyHeadline(textBlock, headlineType);
-                    document.setHeadlines(true);
-                }
-            }
-        } else if (!textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-            PageBlockType headlineType = PageBlockType.getHeadlineType(headlineFontSizes.size() + 1);
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
            headlineClassificationService.classifyHeadline(textBlock, headlineType);
            document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+        } else if (!textBlock.getText().startsWith("Figure ")
+                   && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
+                   && textBlock.getMostPopularWordStyle().equals("bold")
+                   && !document.getFontStyleCounter().getMostPopular().equals("bold")
+                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+            headlineClassificationService.classifyHeadline(textBlock, headlineType);
+            document.setHeadlines(true);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
+                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
+                   && textBlock.getMostPopularWordStyle().equals("bold")
+                   && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
            textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
-                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
-                .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
+                   && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular())
+                   && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular())
+                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
            textBlock.setClassification(PageBlockType.PARAGRAPH);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
+                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
+                   && textBlock.getMostPopularWordStyle().equals("italic")
+                   && !document.getFontStyleCounter().getMostPopular().equals("italic")
+                   && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
            textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
            textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
@@ -0,0 +1,33 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import java.util.regex.Pattern;
+
+public class ClassificationPatterns {
+
+    public static final Pattern HEADLINE_WITH_2_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s(?:14C)?\\s*[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN = Pattern.compile("^([0-9]\\.)\\s[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern AT_LEAST_3_CHARS_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern HEADLINE_PATTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
+
+    public static final Pattern AMOUNT_PATTERN = Pattern.compile(
+            "^\\s*\\d+(?:\\.\\d+)?\\s*(?:ml|ul|μl|l|ug|μg|g|kg|mg|cm|cm2|cm3|mm|mm2|mm3|km|km2|m|m2|m3|lb|oz|ppm|dpm|days|weeks|months|%|f|ppb)\\b",
+            Pattern.CASE_INSENSITIVE);
+
+
+
+    public static final Pattern TABLE_OR_FIGURE_HEADLINE_PATTERN = Pattern.compile(
+            "^\\s*(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
+            Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern TABLE_MID_SENTENCE_PATTERN = Pattern.compile(
+            "(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
+            Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern ALPHANUMERIC = Pattern.compile("[a-zA-Z0-9]");
+
+    public static final Pattern NUMERIC = Pattern.compile("[0-9]+");
+
+}
@@ -0,0 +1,62 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class ClassificationService {
+
+    DocuMineBlockificationService docuMineBlockificationService;
+    BodyTextFrameService bodyTextFrameService;
+    TableOfContentsClassificationService tableOfContentsClassificationService;
+    RedactManagerClassificationService redactManagerClassificationService;
+    ClarifyndClassificationService clarifyndClassificationService;
+    DocuMineClassificationService docuMineClassificationService;
+    HeaderFooterClassificationService headerFooterClassificationService;
+
+
+    public void classify(ClassificationDocument document, LayoutParsingType layoutParsingType, Map<String, String> identifier) {
+
+        log.info("Calculating BodyTextFrame for {}", identifier);
+        bodyTextFrameService.setBodyTextFrames(document, layoutParsingType);
+        for (ClassificationPage page : document.getPages()) {
+            document.getLayoutDebugLayer().addCleanRulingVisualization(page.getCleanRulings(), page.getPageNumber());
+        }
+        log.info("Classify TextBlocks for {}", identifier);
+
+        headerFooterClassificationService.classifyHeadersAndFooters(document);
+
+        tableOfContentsClassificationService.classifyTableOfContents(document);
+
+        switch (layoutParsingType) {
+            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
+                    redactManagerClassificationService.classifyDocument(document);
+            case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(document);
+            case CLARIFYND -> clarifyndClassificationService.classifyDocument(document);
+        }
+
+        if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
+            for (ClassificationPage page : document.getPages()) {
+                docuMineBlockificationService.mergeblocks(page, page.getCleanRulings().withoutTextRulings(), 0, 10);
+            }
+        }
+    }
+
+}
@@ -1,9 +1,21 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.classification;

+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.ALPHANUMERIC;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AMOUNT_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AT_LEAST_3_CHARS_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_PATTERN_WITH_SLASHES;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_2_IDENTIFER_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_MID_SENTENCE_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_OR_FIGURE_HEADLINE_PATTERN;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.regex.Matcher;
-import java.util.regex.Pattern;

 import org.springframework.stereotype.Service;

@@ -11,142 +23,328 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.AbstractBlockOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.HeaderFooterDetection;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DocuMineClassificationService {

-    private static final Pattern HEADLINE_WITH_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern AT_LEAST_3_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern HEADLINE_PATTTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
+    public static final int SEPARATION_THRESHOLD = 10; // if the min distance between a textblock and all its surrounding blocks, the regexes can be more lenient.
+    public static final int SURROUNDING_BLOCKS_RADIUS = 3; // number of surrounding blocks before and after the current textblock to be tested
+
+    ListItemClassificationService listItemClassificationService;


    public void classifyDocument(ClassificationDocument document) {

-        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
-
-        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
+        List<Double> headlineFontSizes = buildHeadlineFontSizes(document);
+        List<AbstractBlockOnPage> blocks = buildBlocksPerPage(document);
+        log.debug("Headline FontSizes are: {}", headlineFontSizes);

        HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();

-        for (ClassificationPage page : document.getPages()) {
-            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
+        for (int i = 0; i < blocks.size(); i++) {
+            AbstractBlockOnPage block = blocks.get(i);
+            document.getLayoutDebugLayer().addTextBlockVisualizations(block.page().getTextBlocks(), block.page().getPageNumber());
+            classifyBlock(headlineClassificationService, i, blocks, document, headlineFontSizes);
        }
-    }

-
-    private void classifyPage(HeadlineClassificationService headlineClassificationService,
-                              ClassificationPage page,
-                              ClassificationDocument document,
-                              List<Float> headlineFontSizes) {
-
-        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
-            if (textBlock instanceof TextPageBlock) {
-                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, page, document, headlineFontSizes);
-            }
-        }
    }


    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
-                               TextPageBlock textBlock,
-                               ClassificationPage page,
+                               int currentIndex,
+                               List<AbstractBlockOnPage> allBlocks,
                               ClassificationDocument document,
-                               List<Float> headlineFontSizes) {
+                               List<Double> headlineFontSizes) {
+
+        TextPageBlock textBlock;
+        if (allBlocks.get(currentIndex).block() instanceof TextPageBlock block) {
+            textBlock = block;
+        } else {
+            return;
+        }
+        ClassificationPage page = allBlocks.get(currentIndex).page();
+        List<AbstractPageBlock> surroundingBlocks = getSurroundingBlocksOnPage(currentIndex, allBlocks);

        log.debug("headlineFontSizes: {}", headlineFontSizes);
        var bodyTextFrame = page.getBodyTextFrame();

-        Matcher headlineWithIdentifierMatcher = HEADLINE_WITH_IDENTIFER_PATTERN.matcher(textBlock.toString());
-        Matcher atLeast3Matcher = AT_LEAST_3_PATTERN.matcher(textBlock.toString());
-        Matcher headlineWithSlashesMatcher = HEADLINE_PATTTERN_WITH_SLASHES.matcher(textBlock.toString());
+        Matcher headlineWith2IdentifierMatcher = HEADLINE_WITH_2_IDENTIFER_PATTERN.matcher(textBlock.toString());
+        Matcher atLeast3Matcher = AT_LEAST_3_CHARS_PATTERN.matcher(textBlock.toString());
+        Matcher headlineWithSlashesMatcher = HEADLINE_PATTERN_WITH_SLASHES.matcher(textBlock.toString());
+        Matcher amountMatcher = AMOUNT_PATTERN.matcher(textBlock.toString());
+        Matcher tableOrFigureMatcher = TABLE_OR_FIGURE_HEADLINE_PATTERN.matcher(textBlock.toString());
+        Matcher tableMidSentenceMatcher = TABLE_MID_SENTENCE_PATTERN.matcher(textBlock.toString());
+        Matcher headlineWithSingleIdentifierMatcher = HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN.matcher(textBlock.toString());
+        boolean isAtLeast3Characters = atLeast3Matcher.reset().find();
+        boolean headlineWithSlashesMatches = headlineWithSlashesMatcher.reset().matches();
+        boolean isAmount = amountMatcher.reset().find();
+        int charCount = countChars(textBlock);
+
+        boolean enoughChars = charCount > textBlock.getText().length() * 0.5;
+
+        List<ListIdentifier> listIdentifiers = listItemClassificationService.findConfirmedListIdentifiers(currentIndex, allBlocks);
+        document.getLayoutDebugLayer().addListIdentifiers(listIdentifiers);

        if (textBlock.getClassification() != null && textBlock.getClassification().isHeadline()) {
            headlineClassificationService.setLastHeadlineFromOutline(textBlock);
            return;
        }
-        if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification(PageBlockType.OTHER);
+        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
+                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
+                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) //
-            || (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) //
-                && (document.getFontSizeCounter().getMostPopular() == null //
-                    || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular()))) {
-            textBlock.setClassification(PageBlockType.HEADER);
+        if (document.getFontSizeCounter().getMostPopular() == null) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+            return;
+        }
+        if (textBlock.getText().length() > 5
+            && greaterOrEqualFontThanDocumentAverage(textBlock, document)
+            && PositionUtils.getApproxLineCount(textBlock) < 5.9
+            && ((textBlock.getMostPopularWordStyle().contains("bold") || textBlock.isUnderlined())//
+                && Character.isDigit(textBlock.toString().charAt(0)) //
+                && isAtLeast3Characters //
+                && !textBlock.toString().contains(":") //
+                || textBlock.toString().startsWith("APPENDIX") //
+                || textBlock.toString().startsWith("FIGURE") //
+                || textBlock.toString().startsWith("Continued TABLE") //
+                || textBlock.toString().startsWith("TABLE"))
+            && !textBlock.toString().endsWith(":")
+            && isAtLeast3Characters
+            && !isAmount
+            && enoughChars) {

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                   || (PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
-                                                          textBlock,
-                                                          page.getRotation())
-                       && (document.getFontSizeCounter().getMostPopular()
-                           == null
-                           || textBlock.getHighestFontSize()
-                              <= document.getFontSizeCounter()
-                                      .getMostPopular()))
-                   || HeaderFooterDetection.isLikelyFooter(textBlock, document, page)) {
-            textBlock.setClassification(PageBlockType.FOOTER);
-        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
-                                                 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
-            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification(PageBlockType.TITLE);
-            }
-        } else if (textBlock.getText().length() > 5
-                   && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular()
-                       || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular())
-                   && PositionUtils.getApproxLineCount(textBlock) < 5.9
+            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
+        } else if (isAllCaps(textBlock)
+                   && ALPHANUMERIC.matcher(Character.toString(textBlock.getText().charAt(0))).matches()
+                   && hasSeparation(textBlock, surroundingBlocks)
+                   && textBlock.getText().length() > 5
+                   && isAtLeast3Characters
+                   && !isAmount
+                   && enoughChars
+                   && !textBlock.toString().contains(":")
+                   && !textBlock.toString().endsWith(".")
+                   && PositionUtils.getApproxLineCount(textBlock) < 2.9) {

-                   && (textBlock.getMostPopularWordStyle().contains("bold")
-                       && Character.isDigit(textBlock.toString().charAt(0))
-                       && atLeast3Matcher.reset().find()
-                       && !textBlock.toString().contains(":") //
-                       || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && atLeast3Matcher.reset().find() && !textBlock.toString().contains(":") //
-                       || textBlock.toString().startsWith("APPENDIX") //
-                       || textBlock.toString().startsWith("FIGURE") //
-                       || textBlock.toString().startsWith("Continued TABLE") //
-                       || textBlock.toString().startsWith("TABLE"))
-                   && !textBlock.toString().endsWith(":")
-                   && atLeast3Matcher.reset().find()) {
-            PageBlockType headlineType = PageBlockType.getHeadlineType(1);
-            headlineClassificationService.classifyHeadline(textBlock, headlineType);
-            document.setHeadlines(true);
-
-        } else if (headlineWithIdentifierMatcher.reset().find()
+            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
+        } else if (headlineWith2IdentifierMatcher.reset().find()
                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && atLeast3Matcher.reset().find()
-                   && !headlineWithSlashesMatcher.reset().matches()) {
-            PageBlockType headlineType = PageBlockType.getHeadlineType(2);
-            headlineClassificationService.classifyHeadline(textBlock, headlineType);
-            document.setHeadlines(true);
+                   && isAtLeast3Characters
+                   && !headlineWithSlashesMatches
+                   && !isAmount) {
+
+            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
+        } else if (hasSeparation(textBlock, surroundingBlocks)//
+                   && greaterOrEqualFontThanPageAverage(textBlock, page)//
+                   && PositionUtils.getApproxLineCount(textBlock) < 2.9//
+                   && (tableOrFigureMatcher.reset().find() || (headlineWithSingleIdentifierMatcher.reset().find() && listIdentifiers.isEmpty())) //
+                   && tableMidSentenceMatcher.reset().results()
+                              .count() <= 1 //
+                   && !isAmount//
+                   && !headlineWithSlashesMatches) {
+
+            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
+//        } else if (textBlock.getMostPopularWordFont().contains("bold")
+//                   && greaterOrEqualFontThanPageAverage(textBlock, page)
+//                   && textBlock.getWords().size() <= 6
+//                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
+//                   && isAtLeast3Characters
+//                   && charCount > textBlock.getText().length() * 0.75
+//                   && !textBlock.getText().contains(":")
+//                   && textBlock.getWidth() < page.getBodyTextFrame().getWidth() * 0.7) {
+//
+//            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
+        } else if (!listIdentifiers.isEmpty()) {
+
+            textBlock.setClassification(PageBlockType.LIST_ITEM);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                   && textBlock.getMostPopularWordStyle().equals("bold")
                   && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+
            textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                   && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular())
                   && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular())
                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+
            textBlock.setClassification(PageBlockType.PARAGRAPH);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                   && textBlock.getMostPopularWordStyle().equals("italic")
                   && !document.getFontStyleCounter().getMostPopular().equals("italic")
                   && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+
            textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
        } else {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
        }
    }

-}
+
+    private int countChars(TextPageBlock textBlock) {
+
+        int count = 0;
+
+        for (int i = 0; i < textBlock.getText().length(); i++) {
+            if (Character.isAlphabetic(textBlock.getText().charAt(i))) {
+                count++;
+            }
+        }
+        return count;
+    }
+
+
+    private static boolean greaterOrEqualFontThanPageAverage(TextPageBlock textBlock, ClassificationPage page) {
+
+        return textBlock.getMostPopularWordHeight() >= page.getTextHeightCounter().getMostPopular() //
+               || textBlock.getMostPopularWordFontSize() >= page.getFontSizeCounter().getMostPopular();
+    }
+
+
+    private static boolean greaterOrEqualFontThanDocumentAverage(TextPageBlock textBlock, ClassificationDocument document) {
+
+        return textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() //
+               || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular();
+    }
+
+
+    private static boolean isAllCaps(TextPageBlock textBlock) {
+
+        return textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT));
+    }
+
+
+    private boolean hasSeparation(TextPageBlock textBlock, List<AbstractPageBlock> surroundingBlocks) {
+
+        return surroundingBlocks.stream()
+                .allMatch(surroundingBlock -> calculateSeparation(textBlock, surroundingBlock) > Math.pow(SEPARATION_THRESHOLD, 2));
+    }
+
+
+    private double calculateMinSeparation(TextPageBlock textBlock, List<AbstractPageBlock> surroundingBlocks) {
+
+        return surroundingBlocks.stream()
+                .mapToDouble(surroundingBlock -> calculateSeparation(textBlock, surroundingBlock))
+                .min().orElse(Double.MAX_VALUE);
+    }
+
+
+    private static double calculateSeparation(TextPageBlock textBlock, AbstractPageBlock surroundingBlock) {
+
+        return Math.pow(surroundingBlock.horizontalDistance(textBlock), 2) + Math.pow(surroundingBlock.verticalDistance(textBlock), 2);
+    }
+
+
+    private static void setAsHeadline(HeadlineClassificationService headlineClassificationService,
+                                      TextPageBlock textBlock,
+                                      ClassificationDocument document,
+                                      List<Double> headlineFontSizes) {
+
+        PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+        headlineClassificationService.classifyHeadline(textBlock, headlineType);
+        document.setHeadlines(true);
+    }
+
+
+    private List<AbstractBlockOnPage> buildBlocksPerPage(ClassificationDocument document) {
+
+        List<AbstractBlockOnPage> blocks = new ArrayList<>();
+        for (ClassificationPage page : document.getPages()) {
+            for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
+                if (abstractPageBlock instanceof TextPageBlock textBlock) {
+                    if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER) //
+                                                                  || textBlock.getClassification().equals(PageBlockType.FOOTER))) {
+                        continue;
+                    }
+                    blocks.add(new AbstractBlockOnPage(textBlock, page));
+                }
+            }
+        }
+        return blocks;
+    }
+
+
+    private static List<Double> buildHeadlineFontSizes(ClassificationDocument document) {
+
+        if (document.getFontSizeCounter().getCountPerValue().size() <= 6) {
+            return document.getFontSizeCounter().getValuesInReverseOrder();
+        }
+
+        List<Map.Entry<Double, Integer>> sortedEntries = new ArrayList<>(document.getFontSizeCounter().getCountPerValue().entrySet());
+        sortedEntries.sort(Map.Entry.comparingByKey());
+
+        int totalCount = sortedEntries.stream()
+                .mapToInt(Map.Entry::getValue).sum();
+
+        int cumulativeCount = 0;
+        Iterator<Map.Entry<Double, Integer>> iterator = sortedEntries.iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<Double, Integer> entry = iterator.next();
+            cumulativeCount += entry.getValue();
+            if (cumulativeCount > totalCount * 0.3) {
+                break; // We've filtered the bottom 30%, so stop.
+            }
+            iterator.remove();
+        }
+
+        if (sortedEntries.size() < 6) {
+            return document.getFontSizeCounter().getValuesInReverseOrder();
+        }
+        int clusterSize = Math.max(1, sortedEntries.size() / 6);
+
+        List<List<Double>> clusters = new ArrayList<>();
+        for (int i = 0; i < 6; i++) {
+            clusters.add(new ArrayList<>());
+        }
+
+        for (int i = 0; i < sortedEntries.size(); i++) {
+            int clusterIndex = Math.min(i / clusterSize, 5);
+            clusters.get(clusterIndex).add(sortedEntries.get(i).getKey());
+        }
+
+        return clusters.stream()
+                .map(cluster -> cluster.stream()
+                        .mapToDouble(d -> d).average()
+                        .orElseThrow())
+                .sorted(Comparator.reverseOrder())
+                .toList();
+    }
+
+
+    private List<AbstractPageBlock> getSurroundingBlocksOnPage(int originalIndex, List<AbstractBlockOnPage> textBlocks) {
+
+        int start = Math.max(originalIndex - SURROUNDING_BLOCKS_RADIUS, 0);
+        int end = Math.min(originalIndex + SURROUNDING_BLOCKS_RADIUS, textBlocks.size());
+        List<AbstractPageBlock> surroundingBlocks = new ArrayList<>(2 * SURROUNDING_BLOCKS_RADIUS);
+        for (int i = start; i < end; i++) {
+            if (i == originalIndex) {
+                continue;
+            }
+            if (textBlocks.get(i).block().getText().length() <= 1) {
+                continue;
+            }
+            if (!textBlocks.get(i).page().equals(textBlocks.get(originalIndex).page())) {
+                continue;
+            }
+            surroundingBlocks.add(textBlocks.get(i).block());
+        }
+        return surroundingBlocks;
+    }
+
+}
+
+
+
@@ -0,0 +1,55 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class HeaderFooterClassificationService {
+
+    public void classifyHeadersAndFooters(ClassificationDocument document) {
+
+        for (ClassificationPage page : document.getPages()) {
+            for (AbstractPageBlock pageBlock : page.getTextBlocks()) {
+                if (pageBlock instanceof TextPageBlock textBlock) {
+                    classifyBlock(document, page, textBlock);
+                }
+            }
+        }
+
+    }
+
+
+    private static void classifyBlock(ClassificationDocument document, ClassificationPage page, TextPageBlock textBlock) {
+
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+            || PositionUtils.isOverBodyTextFrame(page.getBodyTextFrame(), textBlock, page.getRotation()) && smallerFontThanDocAverage(document, textBlock)) {
+
+            textBlock.setClassification(PageBlockType.HEADER);
+
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                   || PositionUtils.isUnderBodyTextFrame(page.getBodyTextFrame(), textBlock, page.getRotation()) && smallerFontThanDocAverage(document, textBlock)) {
+
+            textBlock.setClassification(PageBlockType.FOOTER);
+        }
+    }
+
+
+    private static boolean smallerFontThanDocAverage(ClassificationDocument document, TextPageBlock textBlock) {
+
+        return document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular();
+    }
+
+}
@@ -2,7 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica

 import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.getHeadlineNumber;

+import java.util.List;
+
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;

 import lombok.Getter;
@@ -16,6 +19,7 @@ public class HeadlineClassificationService {
    PageBlockType originalClassifiedBlockType;
    TextPageBlock lastHeadlineFromOutline;

+
    public void setLastHeadlineFromOutline(TextPageBlock lastHeadlineFromOutline) {

        this.lastHeadlineFromOutline = lastHeadlineFromOutline;
@@ -25,28 +29,62 @@ public class HeadlineClassificationService {

    public void classifyHeadline(TextPageBlock textBlock, PageBlockType initialHeadlineType) {

-        TextPageBlock lastHeadline = getLastHeadline();
-        TextPageBlock lastHeadlineFromOutline = getLastHeadlineFromOutline();
-        PageBlockType originalClassifiedBlockType = getOriginalClassifiedBlockType();
        PageBlockType finalHeadlineType = initialHeadlineType;

        if (lastHeadline != null) {

-            if (lastHeadline.equals(lastHeadlineFromOutline)) {
-
-                finalHeadlineType = PageBlockType.getHeadlineType(getHeadlineNumber(lastHeadline.getClassification()) + 1);
-
-            } else if (originalClassifiedBlockType != null && lastHeadline.getClassification() != originalClassifiedBlockType) {
-
-                PageBlockType lastHeadlineType = lastHeadline.getClassification();
-                int difference = getHeadlineNumber(originalClassifiedBlockType) - getHeadlineNumber(lastHeadlineType);
-                finalHeadlineType = PageBlockType.getHeadlineType(getHeadlineNumber(initialHeadlineType) - difference);
-            }
+            finalHeadlineType = decideOnClassification(textBlock, initialHeadlineType);
        }

-        setOriginalClassifiedBlockType(initialHeadlineType);
+        lastHeadline = textBlock;
+        originalClassifiedBlockType = initialHeadlineType;
        textBlock.setClassification(finalHeadlineType);
-        setLastHeadline(textBlock);
+    }
+
+
+    private PageBlockType decideOnClassification(TextPageBlock textBlock, PageBlockType initialHeadlineType) {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText(textBlock.getText());
+        TextPageBlock lastHeadlineFromOutline = getLastHeadlineFromOutline();
+        PageBlockType originalClassifiedBlockType = getOriginalClassifiedBlockType();
+
+        if (!identifier.isEmpty()) {
+            return PageBlockType.getHeadlineType(identifier.level());
+        }
+
+        if (lastHeadline.equals(lastHeadlineFromOutline) && lastHeadline.getMostPopularWordFontSize() >= textBlock.getMostPopularWordFontSize()) {
+
+            return PageBlockType.getHeadlineType(getHeadlineNumber(lastHeadline.getClassification()) + 1);
+
+        } else if (originalClassifiedBlockType != null && lastHeadline.getClassification() != originalClassifiedBlockType) {
+
+            return adjustInitialLevelToLastHeadlineLevel(initialHeadlineType);
+        }
+        return initialHeadlineType;
+    }
+
+
+    private PageBlockType adjustInitialLevelToLastHeadlineLevel(PageBlockType initialHeadlineType) {
+
+        int difference = getHeadlineNumber(originalClassifiedBlockType) - getHeadlineNumber(lastHeadline.getClassification());
+        return PageBlockType.getHeadlineType(Math.max(1, getHeadlineNumber(initialHeadlineType) - difference));
+    }
+
+
+    public static PageBlockType headlineClassByFontSize(TextPageBlock textBlock, List<Double> fontSizeGroups) {
+
+        List<Double> distances = fontSizeGroups.stream()
+                .map(fontSize -> Math.abs(fontSize - textBlock.getMostPopularWordFontSize()))
+                .toList();
+        double min = Double.MAX_VALUE;
+        int argMin = -1;
+        for (int i = 0; i < distances.size(); i++) {
+            if (distances.get(i) < min) {
+                min = distances.get(i);
+                argMin = i;
+            }
+        }
+        return PageBlockType.getHeadlineType(argMin);
    }

 }
@@ -0,0 +1,99 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.AbstractBlockOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+
+@Service
+public class ListItemClassificationService {
+
+    public static final int LIST_IDENTIFIER_SEARCH_RADIUS = 3;
+
+
+    public List<ListIdentifier> findConfirmedListIdentifiers(int currentIndex, List<AbstractBlockOnPage> allBlocks) {
+
+        List<ListIdentifier> listIdentifiers = extractListIdentifiers(allBlocks.get(currentIndex));
+        if (listIdentifiers.isEmpty()) {
+            return Collections.emptyList();
+        }
+        if (listIdentifiers.size() > 1 && ListIdentifier.isInOrder(listIdentifiers)) {
+            return listIdentifiers;
+        }
+
+        int start = Math.max(0, currentIndex - LIST_IDENTIFIER_SEARCH_RADIUS);
+        int end = Math.min(allBlocks.size(), currentIndex + LIST_IDENTIFIER_SEARCH_RADIUS);
+
+        List<ListIdentifier> identifiersBehind = new ArrayList<>();
+        if (start < currentIndex) {
+            identifiersBehind.addAll(allBlocks.subList(start, currentIndex)
+                                             .stream()
+                                             .map(this::extractListIdentifiers)
+                                             .flatMap(Collection::stream)
+                                             .toList());
+        }
+        if (!identifiersBehind.isEmpty()) {
+            listIdentifiers.add(0, identifiersBehind.get(identifiersBehind.size() - 1));
+            if (ListIdentifier.isInOrder(listIdentifiers)) {
+                return listIdentifiers;
+            }
+            listIdentifiers.remove(0);
+        }
+        List<ListIdentifier> identifiersAhead = new ArrayList<>();
+        if (currentIndex + 1 < end) {
+            identifiersAhead.addAll(allBlocks.subList(currentIndex + 1, end)
+                                            .stream()
+                                            .map(this::extractListIdentifiers)
+                                            .flatMap(Collection::stream)
+                                            .toList());
+        }
+        if (!identifiersAhead.isEmpty()) {
+            listIdentifiers.add(identifiersAhead.get(0));
+            if (ListIdentifier.isInOrder(listIdentifiers)) {
+                return listIdentifiers;
+            }
+            listIdentifiers.remove(listIdentifiers.size() - 1);
+        }
+        return Collections.emptyList();
+
+    }
+
+
+    private List<ListIdentifier> extractListIdentifiers(AbstractBlockOnPage block) {
+
+        List<ListIdentifier> result = new LinkedList<>();
+        if (block.block() instanceof TextPageBlock textBlock) {
+            List<Word> sequences = textBlock.getWords();
+            for (int i = 0; i < sequences.size(); i++) {
+
+                if (i != 0 && sequences.get(i - 1).getXDirAdj() < sequences.get(i).getXDirAdj()) {
+                    // is not the start of a line, continue
+                    continue;
+                }
+
+                Word sequence = sequences.get(i);
+                List<Word> wordsAtStartOfLine = new ArrayList<>(3);
+                int end = Math.min(sequences.size(), i + 3);
+                for (int j = i; j < end; j++) {
+                    if (sequences.get(j).intersectsYDirAdj(sequence, 2)) {
+                        wordsAtStartOfLine.add(sequences.get(j));
+                    } else {
+                        break;
+                    }
+                }
+
+                ListIdentifier.parse(wordsAtStartOfLine, block.page().getPageNumber()).ifPresent(result::add);
+            }
+        }
+        return result;
+    }
+
+}
@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+
+public record NumberWord(Word word, int number) {
+
+}
@@ -11,7 +11,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -22,10 +21,9 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RedactManagerClassificationService {

-
    public void classifyDocument(ClassificationDocument document) {

-        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

@@ -37,7 +35,10 @@ public class RedactManagerClassificationService {
    }


-    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService,
+                              ClassificationPage page,
+                              ClassificationDocument document,
+                              List<Double> headlineFontSizes) {

        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextPageBlock) {
@@ -47,7 +48,11 @@ public class RedactManagerClassificationService {
    }


-    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
+                               TextPageBlock textBlock,
+                               ClassificationPage page,
+                               ClassificationDocument document,
+                               List<Double> headlineFontSizes) {

        var bodyTextFrame = page.getBodyTextFrame();

@@ -55,8 +60,17 @@ public class RedactManagerClassificationService {
            headlineClassificationService.setLastHeadlineFromOutline(textBlock);
            return;
        }
+        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
+                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
+                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
+            return;
+        }
        if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+            return;
+        }
+        if (document.getFontSizeCounter().getMostPopular() == null) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
            return;
        }

@@ -67,69 +81,58 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.PARAGRAPH);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-            || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
-                                                                                                   || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-            textBlock.setClassification(PageBlockType.HEADER);
+        if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
+                                          && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                   || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
-                                                                                                           || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-            textBlock.setClassification(PageBlockType.FOOTER);
-        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
-                                                 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification(PageBlockType.TITLE);
+                PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+                headlineClassificationService.classifyHeadline(textBlock, headlineType);
+                document.setHeadlines(true);
            }
        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
                   && PositionUtils.getApproxLineCount(textBlock) < 4.9
                   && (textBlock.getMostPopularWordStyle().equals("bold")
                       || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                          && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getSequences()
-                              .get(0).getTextPositions()
-                              .get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {

-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                    PageBlockType headlineType = PageBlockType.getHeadlineType(i);
-                    headlineClassificationService.classifyHeadline(textBlock, headlineType);
-                    document.setHeadlines(true);
-                }
-            }
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+            headlineClassificationService.classifyHeadline(textBlock, headlineType);
+            document.setHeadlines(true);
        } else if (!textBlock.getText().startsWith("Figure ")
                   && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                   && textBlock.getMostPopularWordStyle().equals("bold")
                   && !document.getFontStyleCounter().getMostPopular().equals("bold")
                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getSequences()
-                              .get(0).getTextPositions()
-                              .get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-            PageBlockType headlineType = PageBlockType.getHeadlineType(headlineFontSizes.size() + 1);
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
            headlineClassificationService.classifyHeadline(textBlock, headlineType);
            document.setHeadlines(true);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                   && textBlock.getMostPopularWordStyle().equals("bold")
                   && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+
            textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                   && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular())
                   && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular())
                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+
            textBlock.setClassification(PageBlockType.PARAGRAPH);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                   && textBlock.getMostPopularWordStyle().equals("italic")
                   && !document.getFontStyleCounter().getMostPopular().equals("italic")
                   && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+
            textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
+
            textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
        } else {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
        }
    }

@@ -0,0 +1,427 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AMOUNT_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.NUMERIC;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextBlockOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TocNumberComparator;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
+import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class TableOfContentsClassificationService {
+
+    private static final int MAX_PAGE_COUNT = 10; // maximum length of a toc to avoid runaway classification
+    private static final int SURROUNDING_BLOCKS_RADIUS = 10; // number of blocks to look ahead
+    private static final int MINIMUM_MATCHES = 2; // minimum cluster size
+    public static final int INTERSECTION_TOLERANCE = 2; // equality threshold for x intersection
+    public static final int DENSITY_THRESHOLD_COUNT = 10; // describes the minimum density, at least this many entries per page height are required
+
+
+    @SuppressWarnings("checkstyle:ModifiedControlVariable")
+    public void classifyTableOfContents(ClassificationDocument document) {
+
+        List<TextBlockOnPage> textBlocks = buildBlocksPerPage(document);
+
+        for (int i = 0; i < textBlocks.size(); i++) {
+            TextBlockOnPage textBlock = textBlocks.get(i);
+
+            if (!isTOCHeadline(textBlock)) {
+                continue;
+            }
+
+            int end = identifyTOCItems(i + 1, textBlocks, document);
+
+            if (end > i + 1) {
+                if (textBlock.textBlock().getClassification() == null) {
+                    textBlock.textBlock().setClassification(PageBlockType.H1);
+                }
+                i = end;
+            }
+        }
+    }
+
+
+    private int identifyTOCItems(int start, List<TextBlockOnPage> textBlocks, ClassificationDocument document) {
+
+        if (start >= textBlocks.size()) {
+            return start;
+        }
+        ClassificationPage startPage = textBlocks.get(start).page();
+        List<TextBlockOnPage> initialLookAhead = textBlocks.subList(start, Math.min(start + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()));
+        HashMap<NumberWord, TextBlockOnPage> numberToBlockLookup = new HashMap<>();
+        List<NumberWord> numbers = extractNumbers(initialLookAhead, numberToBlockLookup, document.getPages().size());
+        TocNumberFinder tocNumberFinder = new TocNumberFinder(numbers, numberToBlockLookup);
+
+        int lastCandidate = start;
+        for (int i = start; i < Math.min(lastCandidate + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()); i++) {
+
+            TextBlockOnPage textBlockOnPage = textBlocks.get(i);
+            if (textBlockOnPage.page().getPageNumber() - MAX_PAGE_COUNT > startPage.getPageNumber()) {
+                break;
+            }
+
+            if (textBlockOnPage.textBlock().getClassification() != null //
+                && textBlockOnPage.textBlock().getClassification().isHeadline() //
+                && !(textBlockOnPage.textBlock().getText().startsWith("TABLES") //
+                     || textBlockOnPage.textBlock().getText().startsWith("APPENDICES") //
+                     || textBlockOnPage.textBlock().getText().startsWith("FIGURES"))) {
+                log.debug("hit an outline headline, stop immediately.");
+                lastCandidate = i - 1;
+                break;
+            }
+
+            List<NumberWord> numbersFromBlock = extractNumbers(textBlockOnPage, numberToBlockLookup, document.getPages().size());
+
+            List<NumberWord> currentRightmostCluster = tocNumberFinder.getCurrentRightmostCluster();
+
+            if (currentRightmostCluster.size() < MINIMUM_MATCHES) {
+                log.debug("No numbers indicating a table of contents here.");
+                return start;
+            }
+
+            if (anyIntersection(currentRightmostCluster, numbersFromBlock, numberToBlockLookup)) {
+                lastCandidate = i;
+                numbersFromBlock.forEach(tocNumberFinder::add);
+            }
+        }
+
+        Set<TextBlockOnPage> blocksWithNumberInCluster = tocNumberFinder.getCurrentRightmostCluster()
+                .stream()
+                .map(numberToBlockLookup::get)
+                .collect(Collectors.toSet());
+
+        addVisualization(document.getLayoutDebugLayer(), tocNumberFinder, numberToBlockLookup, blocksWithNumberInCluster, textBlocks.get(start - 1));
+
+        int lastConfirmed = start;
+        for (int i = start; i < lastCandidate + 1; i++) {
+            TextBlockOnPage textBlockOnPage = textBlocks.get(i);
+            if (blocksWithNumberInCluster.contains(textBlockOnPage)) {
+                lastConfirmed = i;
+            }
+        }
+
+        textBlocks.subList(start, lastConfirmed + 1)
+                .stream()
+                .filter(block -> (block.textBlock().getClassification() == null || !block.textBlock().getClassification().isHeadline()))
+                .forEach(textBlockOnPage -> textBlockOnPage.textBlock().setClassification(PageBlockType.TABLE_OF_CONTENTS_ITEM));
+
+        return lastCandidate;
+    }
+
+
+    private static void addVisualization(LayoutDebugLayer layoutDebugLayer,
+                                         TocNumberFinder tocNumberFinder,
+                                         Map<NumberWord, TextBlockOnPage> lookup,
+                                         Set<TextBlockOnPage> blocksWithNumberInCluster,
+                                         TextBlockOnPage startingHeadline) {
+
+        tocNumberFinder.getCurrentRightmostCluster()
+                .stream()
+                .collect(Collectors.groupingBy(key -> lookup.get(key).page().getPageNumber()))
+                .forEach((pageNumber, number) -> layoutDebugLayer.addTocPages(number, pageNumber));
+        layoutDebugLayer.addTocBlocks(blocksWithNumberInCluster);
+        layoutDebugLayer.addTocBlocks(Set.of(startingHeadline));
+    }
+
+
+    private static boolean anyIntersection(Collection<NumberWord> numbers1, Collection<NumberWord> numbers2, Map<NumberWord, TextBlockOnPage> lookup) {
+
+        return numbers1.stream()
+                .anyMatch(numberFromCluster -> numbers2.stream()
+                        .anyMatch(numberFromBlock -> matches(numberFromBlock, numberFromCluster, lookup)));
+    }
+
+
+    private static List<NumberWord> extractNumbers(List<TextBlockOnPage> textBlocks, Map<NumberWord, TextBlockOnPage> lookup, int numberOfPages) {
+
+        List<NumberWord> blocks = new LinkedList<>();
+        for (TextBlockOnPage textBlock : textBlocks) {
+            blocks.addAll(extractNumbers(textBlock, lookup, numberOfPages));
+        }
+        return blocks;
+    }
+
+
+    private static List<NumberWord> extractNumbers(TextBlockOnPage textBlock, Map<NumberWord, TextBlockOnPage> lookup, int numberOfPages) {
+
+        List<NumberWord> blocks = new LinkedList<>();
+        TextPageBlock block = textBlock.textBlock();
+        List<Word> words = block.getWords();
+        for (int i = 0; i < words.size(); i++) {
+
+            Word word = words.get(i);
+            if (!wordIsEndOfLine(i, words)) {
+                continue;
+            }
+
+            if (AMOUNT_PATTERN.matcher(getSurroundingString(i, words)).matches()) {
+                continue;
+            }
+
+            Matcher matcher = SectionIdentifier.numericalIdentifierPattern.matcher(word.toString());
+            if (matcher.find() && matcher.group(2) != null) {
+                continue;
+            }
+
+            Matcher numberFinder = NUMERIC.matcher(word);
+            if (!numberFinder.find() || word.length() > 5) {
+                continue;
+            }
+
+            try {
+                int pageNumber = Integer.parseInt(numberFinder.group());
+                if (0 >= pageNumber || pageNumber > numberOfPages) {
+                    continue;
+                }
+                NumberWord numberWord = new NumberWord(word, pageNumber);
+                lookup.put(numberWord, textBlock);
+                blocks.add(numberWord);
+            } catch (NumberFormatException e) {
+                log.debug("That wasn't a number! Should not happen, due to numeric check beforehand.");
+            }
+        }
+        return blocks;
+    }
+
+
+    private static boolean wordIsEndOfLine(int i, List<Word> words) {
+
+        if (i == words.size() - 1) {
+            return true;
+        }
+        Word word = words.get(i);
+        Word nextWord = words.get(i + 1);
+        return !nextWord.rightOf(word);
+    }
+
+
+    private static CharSequence getSurroundingString(int i, List<Word> sequences) {
+
+        int end = Math.min(i + 5, sequences.size());
+        return sequences.subList(i, end)
+                .stream()
+                .map(Word::toString)
+                .collect(Collectors.joining(" "));
+    }
+
+
+    private static boolean matches(NumberWord number1, NumberWord number2, Map<NumberWord, TextBlockOnPage> lookup) {
+
+        if (number1.word().getDir() != number2.word().getDir()) {
+            return false;
+        }
+
+        return number1.word().intersectsXDirAdj(number2.word(), INTERSECTION_TOLERANCE);
+    }
+
+
+    private boolean isTOCHeadline(TextBlockOnPage textBlock) {
+
+        if (textBlock.textBlock().getText().length() > 50) {
+            return false;
+        }
+        String text = TextNormalizationUtilities.removeAllWhitespaces(textBlock.textBlock().getText().toLowerCase(Locale.ENGLISH));
+        return (text.contains("content") && text.length() < "content".length() + 6) //
+               || (text.contains("tableofcontent") && text.length() < "tableofcontent".length() + DENSITY_THRESHOLD_COUNT)//
+               || text.equals("tables")//
+               || text.equals("appendices")//
+               || text.equals("figures");
+    }
+
+
+    private List<TextBlockOnPage> buildBlocksPerPage(ClassificationDocument document) {
+
+        List<TextBlockOnPage> blocks = new ArrayList<>();
+        for (ClassificationPage page : document.getPages()) {
+            for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
+                if (abstractPageBlock instanceof TextPageBlock textBlock) {
+                    if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER) //
+                                                                  || textBlock.getClassification().equals(PageBlockType.FOOTER))) {
+                        continue;
+                    }
+                    blocks.add(new TextBlockOnPage(textBlock, page));
+                }
+            }
+        }
+        return blocks;
+    }
+
+
+    private static class TocNumberFinder {
+
+        final UnionFind<NumberWord> numberClusters;
+        final HashMap<NumberWord, TextBlockOnPage> lookup;
+
+
+        TocNumberFinder(List<NumberWord> blocks, HashMap<NumberWord, TextBlockOnPage> lookup) {
+
+            this.numberClusters = new UnionFind<>(new HashSet<>(blocks));
+            for (int i = 0; i < blocks.size(); i++) {
+                for (int j = i + 1; j < blocks.size(); j++) {
+                    if (matches(blocks.get(i), blocks.get(j), lookup)) {
+                        numberClusters.union(blocks.get(i), blocks.get(j));
+                    }
+                }
+            }
+            this.lookup = lookup;
+        }
+
+
+        public void add(NumberWord number) {
+
+            if (numberClusters.getElements().contains(number)) {
+                return;
+            }
+
+            numberClusters.addElement(number);
+            for (NumberWord element : numberClusters.getElements()) {
+                if (matches(number, element, lookup)) {
+                    numberClusters.union(element, number);
+                }
+            }
+        }
+
+
+        public List<NumberWord> getCurrentRightmostCluster() {
+
+            return numberClusters.getGroups()
+                    .stream()
+                    .filter(cluster -> cluster.size() > MINIMUM_MATCHES)
+                    .map(cluster -> cluster.stream()
+                            .sorted(new TocNumberComparator(lookup))
+                            .toList())
+                    .map(this::removeOutliers)
+                    .map(this::removeOnNonConsecutivePages)
+                    .map(this::filterByWordNearTopOfPage)
+                    .filter(cluster -> cluster.size() > MINIMUM_MATCHES)
+                    .max(Comparator.comparingDouble(cluster -> cluster.get(0).word().getBBox().getMaxX())).orElse(Collections.emptyList());
+        }
+
+
+        private List<NumberWord> removeOnNonConsecutivePages(List<NumberWord> numbers) {
+
+            List<NumberWord> result = new ArrayList<>();
+
+            result.add(numbers.get(0));
+
+            for (int i = 1; i < numbers.size(); i++) {
+                int prev = getPageNumber(numbers, i - 1);
+                int curr = getPageNumber(numbers, i);
+
+                if (Math.abs(prev - curr) > 1) {
+                    break;
+                } else {
+                    result.add(numbers.get(i));
+                }
+            }
+            return result;
+        }
+
+
+        private int getPageNumber(List<NumberWord> numbers, int i) {
+
+            return lookup.get(numbers.get(i)).page().getPageNumber();
+        }
+
+
+        private List<NumberWord> filterByWordNearTopOfPage(List<NumberWord> numbers) {
+
+            List<NumberWord> result = new ArrayList<>();
+
+            result.add(numbers.get(0));
+
+            for (int i = 1; i < numbers.size(); i++) {
+                NumberWord prev = numbers.get(i - 1);
+                NumberWord curr = numbers.get(i);
+                ClassificationPage prevPage = lookup.get(prev).page();
+                ClassificationPage currPage = lookup.get(curr).page();
+                if (prevPage.equals(currPage)) {
+                    result.add(curr);
+                } else if (curr.word().getBBox().getMinY() < currPage.getPageHeight() * 0.33) {
+                    result.add(curr);
+                }
+            }
+            return result;
+        }
+
+
+        public List<NumberWord> removeOutliers(List<NumberWord> numbers) {
+
+            List<NumberWord> confirmedClusterNumbers = new ArrayList<>();
+
+            confirmedClusterNumbers.add(numbers.get(0));
+
+            for (int i = 1; i < numbers.size() - 1; i++) {
+                int prev = getNumberAsInt(numbers, i - 1);
+                int curr = getNumberAsInt(numbers, i);
+                int next = getNumberAsInt(numbers, i + 1);
+
+                if (!(curr <= prev || curr >= next) || !isBetterWithout(numbers, i)) {
+                    confirmedClusterNumbers.add(numbers.get(i));
+                }
+            }
+            if (getNumberAsInt(numbers, numbers.size() - 1) >= getLatestNumber(confirmedClusterNumbers)) {
+                confirmedClusterNumbers.add(numbers.get(numbers.size() - 1));
+            }
+
+            return confirmedClusterNumbers;
+        }
+
+
+        private static int getLatestNumber(List<NumberWord> confirmedClusterNumbers) {
+
+            return confirmedClusterNumbers.get(confirmedClusterNumbers.size() - 1).number();
+        }
+
+
+        // Helper method to check if removing the current number results in a better order
+        public static boolean isBetterWithout(List<NumberWord> numbers, int i) {
+
+            if (i == 0 || i == numbers.size() - 1) {
+                return false;
+            }
+
+            int prev = getNumberAsInt(numbers, i - 1);
+            int curr = getNumberAsInt(numbers, i);
+            int next = getNumberAsInt(numbers, i + 1);
+
+            return (prev <= next) && (Math.abs(prev - next) < Math.abs(prev - curr) + Math.abs(curr - next));
+        }
+
+    }
+
+
+    private static int getNumberAsInt(List<NumberWord> numbers, int i) {
+
+        return numbers.get(i).number();
+    }
+
+}
@@ -6,6 +6,7 @@ import static java.util.stream.Collectors.toList;

 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -15,6 +16,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
@@ -32,7 +34,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.He
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableOfContentItem;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
@@ -68,21 +72,37 @@ public class DocumentGraphFactory {
        documentGraph.setPages(context.pages.keySet());
        documentGraph.setDocumentTree(context.documentTree);
        documentGraph.setTextBlock(documentGraph.getTextBlock());
+        addTextBlocksToPages(documentGraph);
+
        return documentGraph;
    }


+    private void addTextBlocksToPages(Document documentGraph) {
+
+        documentGraph.streamAllSubNodes()
+                .filter(SemanticNode::isLeaf)
+                .filter(node -> !node.getType().equals(NodeType.HEADER))
+                .filter(node -> !node.getType().equals(NodeType.FOOTER))
+                .filter(node -> !node.getType().equals(NodeType.IMAGE))
+                .map(SemanticNode::getTextBlock)
+                .map(TextBlock::getAtomicTextBlocks)
+                .flatMap(Collection::stream)
+                .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
+    }
+
+
    private void addSections(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument, Context context, Document document) {

        for (TableOfContentItem tocItem : classificationDocument.getTableOfContents()) {
-            var parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
-            Optional<AbstractSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
-                                                                                   parent,
-                                                                                   tocItem.getChildren().isEmpty(),
-                                                                                   tocItem.getNonEmptySectionBlocks(),
-                                                                                   tocItem.getImages(),
-                                                                                   context,
-                                                                                   document);
+            GenericSemanticNode parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
+            Optional<GenericSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
+                                                                                  parent,
+                                                                                  tocItem.getChildren().isEmpty(),
+                                                                                  tocItem.getNonEmptySectionBlocks(),
+                                                                                  tocItem.getImages(),
+                                                                                  context,
+                                                                                  document);
            tocItem.setSection(section.orElse(null));
        }
    }
@@ -105,17 +125,15 @@ public class DocumentGraphFactory {
            node = Paragraph.builder().documentTree(context.getDocumentTree()).build();
        }

-        page.getMainBody().add(node);
-
        List<TextPageBlock> textBlocks = new ArrayList<>();
        textBlocks.add(originalTextBlock);
        textBlocks.addAll(textBlocksToMerge);

-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSort(textBlocks), node, context, page);

        if (node instanceof DuplicatedParagraph duplicatedParagraph) {
            AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock(textBlocks.stream()
-                                                                                                      .flatMap(tb -> tb.getSequences()
+                                                                                                      .flatMap(tb -> tb.getWords()
                                                                                                              .stream())
                                                                                                      .collect(Collectors.toList()), node, context, page);
            duplicatedParagraph.setUnsortedLeafTextBlock(unsortedTextBlock);
@@ -141,7 +159,7 @@ public class DocumentGraphFactory {

        Rectangle2D position = image.getPosition();
        Page page = context.getPage(image.getPage());
-        Image imageNode = Image.builder()
+        return Image.builder()
                .id(IdBuilder.buildId(Set.of(page), List.of(position)))
                .imageType(image.getImageType())
                .position(position)
@@ -150,8 +168,6 @@ public class DocumentGraphFactory {
                .representationHash(image.getRepresentation())
                .documentTree(context.getDocumentTree())
                .build();
-        page.getMainBody().add(imageNode);
-        return imageNode;
    }


@@ -191,7 +207,7 @@ public class DocumentGraphFactory {

        Page page = context.getPage(textBlocks.get(0).getPage());
        Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeTextPositionSequence(textBlocks), footer, context, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.merge(textBlocks), footer, context, page);
        List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer);
        footer.setTreeId(tocId);
        footer.setLeafTextBlock(textBlock);
@@ -203,7 +219,7 @@ public class DocumentGraphFactory {

        Page page = context.getPage(textBlocks.get(0).getPage());
        Header header = Header.builder().documentTree(context.getDocumentTree()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeTextPositionSequence(textBlocks), header, 0, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.merge(textBlocks), header, 0, page);
        List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header);
        header.setTreeId(tocId);
        header.setLeafTextBlock(textBlock);
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.factory;

-import java.awt.geom.AffineTransform;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition.HEIGHT_PADDING;
+
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.Collections;
@@ -11,25 +12,23 @@ import java.util.Objects;

 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@UtilityClass
 public class SearchTextWithTextPositionFactory {

-    public final int HEIGHT_PADDING = 2;
    // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away.
    // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height.
    // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate
    // This is why, we need to initialize this to < -2, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there.
    // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3.
    public final int MAX_HYPHEN_LINEBREAK_DISTANCE = 3;
-    public static final double LINEBREAK_DELTA_TOLERANCE = 1.05;
+    public static final double LINEBREAK_DELTA_TOLERANCE = 1.5;


-    public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<TextPositionSequence> sequences) {
+    public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<Word> sequences) {

        if (sequences.isEmpty() || sequences.stream()
                .allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
@@ -38,15 +37,13 @@ public class SearchTextWithTextPositionFactory {

        Context context = new Context();

-        RedTextPosition currentTextPosition = sequences.get(0).getTextPositions()
-                .get(0);
+        RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
        RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").bBoxDirAdj(currentTextPosition.getBBoxDirAdj()).build();

-        for (TextPositionSequence word : sequences) {
+        for (Word word : sequences) {
            for (int i = 0; i < word.getTextPositions().size(); ++i) {

-                currentTextPosition = word.getTextPositions()
-                        .get(i);
+                currentTextPosition = word.getTextPositions().get(i);
                if (isLineBreak(currentTextPosition, previousTextPosition)) {
                    removeHyphenLinebreaks(context);
                    context.lineBreaksStringIdx.add(context.stringIdx);
@@ -68,11 +65,10 @@ public class SearchTextWithTextPositionFactory {
            ++context.stringIdx;
        }

-
        List<Rectangle2D> positions = sequences.stream()
-                .map(TextPositionSequence::getTextPositions)
+                .map(Word::getTextPositions)
                .flatMap(Collection::stream)
-                .map(RedTextPosition::getBBoxInitialUserSpace)
+                .map(RedTextPosition::getBBoxPdf)
                .toList();

        assert context.stringBuilder.length() == context.stringIdxToPositionIdx.size();
@@ -161,8 +157,8 @@ public class SearchTextWithTextPositionFactory {
            return false;
        }

-        double deltaY = Math.abs(currentPosition.getYDirAdj() - previousPosition.getYDirAdj()) * LINEBREAK_DELTA_TOLERANCE;
-        return deltaY >= currentPosition.getHeightDir() || deltaY >= previousPosition.getHeightDir();
+        double deltaY = (Math.abs(currentPosition.getYDirAdj() - previousPosition.getYDirAdj()) * LINEBREAK_DELTA_TOLERANCE) + (2 * HEIGHT_PADDING);
+        return deltaY >= currentPosition.getHeightDirAdj() || deltaY >= previousPosition.getHeightDirAdj();
    }


@@ -188,32 +184,6 @@ public class SearchTextWithTextPositionFactory {
    }


-    public Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
-
-        float textHeight = sequence.getTextHeight() + HEIGHT_PADDING;
-        Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(),
-                textPosition.getYDirAdj() - textHeight,
-                textPosition.getWidthDirAdj(),
-                textHeight + HEIGHT_PADDING);
-
-        AffineTransform transform = new AffineTransform();
-
-        if (sequence.getDir() == TextDirection.ZERO || sequence.getDir() == TextDirection.HALF_CIRCLE) {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageWidth() / 2f, sequence.getPageHeight() / 2f);
-            transform.translate(0f, sequence.getPageHeight());
-        } else if (sequence.getDir() == TextDirection.QUARTER_CIRCLE) {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageWidth() / 2f, sequence.getPageWidth() / 2f);
-            transform.translate(0f, sequence.getPageWidth());
-        } else {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageHeight() / 2f, sequence.getPageHeight() / 2f);
-            transform.translate(0f, sequence.getPageWidth());
-        }
-        transform.scale(1., -1.);
-
-        return transform.createTransformedShape(rectangle2D).getBounds2D();
-    }
-
-
    private class Context {

        List<Integer> stringIdxToPositionIdx = new LinkedList<>();
@@ -2,13 +2,11 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;

 import static java.lang.String.format;
 import static java.util.Collections.emptyList;
-import static java.util.stream.Collectors.groupingBy;

 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;

@@ -17,7 +15,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
@@ -30,13 +27,13 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class SectionNodeFactory {

-    public Optional<AbstractSemanticNode> addSection(LayoutParsingType layoutParsingType,
-                                                     GenericSemanticNode parentNode,
-                                                     boolean isLeaf,
-                                                     List<AbstractPageBlock> pageBlocks,
-                                                     List<ClassifiedImage> images,
-                                                     DocumentGraphFactory.Context context,
-                                                     Document document) {
+    public Optional<GenericSemanticNode> addSection(LayoutParsingType layoutParsingType,
+                                                    GenericSemanticNode parentNode,
+                                                    boolean isLeaf,
+                                                    List<AbstractPageBlock> pageBlocks,
+                                                    List<ClassifiedImage> images,
+                                                    DocumentGraphFactory.Context context,
+                                                    Document document) {

        // This is for the case where we have images on a page without any text/footer/header.
        // The pageBlocks list is empty, but we still need to add those images to the document.
@@ -51,24 +48,19 @@ public class SectionNodeFactory {
            return Optional.empty();
        }

-        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream()
-                .collect(groupingBy(AbstractPageBlock::getPage));
-
        AbstractSemanticNode section;
-        boolean containsTablesAndTextBlocks = containsTablesAndTextBlocks(pageBlocks);
-        if (isLeaf && !containsTablesAndTextBlocks) {
+        if (isLeaf) {
            section = Section.builder().documentTree(context.getDocumentTree()).build();
        } else {
            section = SuperSection.builder().documentTree(context.getDocumentTree()).build();
        }

        context.getSections().add(section);
-        blocksPerPage.keySet()
-                .forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));

        section.setTreeId(getTreeId(parentNode, context, section));

        addFirstHeadlineDirectlyToSection(layoutParsingType, pageBlocks, context, section, document);
+        boolean containsTablesAndTextBlocks = containsTablesAndTextBlocks(pageBlocks);
        if (containsTablesAndTextBlocks) {
            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(layoutParsingType,
                                                                                                  section,
@@ -158,7 +150,8 @@ public class SectionNodeFactory {
    private boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {

        return pageBlocks.stream()
-                       .anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
+                       .anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) //
+               && pageBlocks.stream()
                       .anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
    }

@@ -234,18 +227,11 @@ public class SectionNodeFactory {
                .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
                .filter(abstractTextContainer -> abstractTextContainer instanceof TextPageBlock)
-                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
+                .filter(abstractTextContainer -> abstractTextContainer.intersectsYPdf(atc))
                .map(abstractTextContainer -> (TextPageBlock) abstractTextContainer)
                .filter(abstractTextContainer -> abstractTextContainer.getDir() == atc.getDir())
                .filter(abstractTextContainer -> !abstractTextContainer.isToDuplicate())
                .toList();
    }

-
-    private void addSectionNodeToPageNode(DocumentGraphFactory.Context context, AbstractSemanticNode section, Integer pageNumber) {
-
-        Page page = context.getPage(pageNumber);
-        page.getMainBody().add(section);
-    }
-
 }
@@ -4,7 +4,6 @@ import static java.util.Collections.emptyList;

 import java.util.Collection;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@@ -12,13 +11,12 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import lombok.experimental.UtilityClass;
@@ -36,10 +34,7 @@ public class TableNodeFactory {
                         Document document) {

        setPageNumberInCells(tablesToMerge);
-        Set<Page> pages = tablesToMerge.stream()
-                .map(AbstractPageBlock::getPage)
-                .map(context::getPage)
-                .collect(Collectors.toSet());
+
        List<List<Cell>> mergedRows = tablesToMerge.stream()
                .map(TablePageBlock::getRows)
                .flatMap(Collection::stream)
@@ -51,8 +46,6 @@ public class TableNodeFactory {
                .numberOfRows(mergedRows.size())
                .build();

-        pages.forEach(page -> addTableToPage(page, parentNode, table));
-
        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table);
        table.setTreeId(treeId);
        addTableCells(layoutParsingType, mergedRows, table, context, document);
@@ -82,17 +75,6 @@ public class TableNodeFactory {
    }


-    @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong
-    private void addTableToPage(Page page, SemanticNode parentNode, Table table) {
-
-        if (!page.getMainBody().contains(parentNode)) {
-            parentNode.getPages().add(page);
-        }
-
-        page.getMainBody().add(table);
-    }
-
-
    private void ifTableHasNoHeadersSetFirstRowAsHeaders(Table table) {

        if (table.streamHeaders()
@@ -107,14 +89,7 @@ public class TableNodeFactory {

        for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
            for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(layoutParsingType,
-                             rows.get(rowIndex)
-                                     .get(colIndex),
-                             rowIndex,
-                             colIndex,
-                             table,
-                             context,
-                             document);
+                addTableCell(layoutParsingType, rows.get(rowIndex).get(colIndex), rowIndex, colIndex, table, context, document);
            }
        }
    }
@@ -131,14 +106,7 @@ public class TableNodeFactory {

        Page page = context.getPage(cell.getPageNumber());

-        TableCell tableCell = TableCell.builder()
-                .documentTree(context.getDocumentTree())
-                .row(rowIndex)
-                .col(colIndex)
-                .header(cell.isHeaderCell())
-                .bBox(cell.getBBoxInitialUserSpace())
-                .build();
-        page.getMainBody().add(tableCell);
+        TableCell tableCell = TableCell.builder().documentTree(context.getDocumentTree()).row(rowIndex).col(colIndex).header(cell.isHeaderCell()).bBox(cell.getBBoxPdf()).build();

        List<Integer> treeId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell);
        tableCell.setTreeId(treeId);
@@ -147,9 +115,7 @@ public class TableNodeFactory {
        if (cell.getTextBlocks().isEmpty()) {
            tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page));
        } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.getTextBlockFactory()
-                    .buildAtomicTextBlock(cell.getTextBlocks()
-                                                  .get(0).getSequences(), tableCell, context, page);
+            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getWords(), tableCell, context, page);
            tableCell.setLeafTextBlock(textBlock);
        } else if (firstTextBlockIsHeadline(cell)) {
            SectionNodeFactory.addSection(layoutParsingType,
@@ -163,7 +129,7 @@ public class TableNodeFactory {
                                          context,
                                          document);
        } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
-            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
+            List<Word> sequences = TextPositionOperations.mergeAndSort(cell.getTextBlocks());
            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
            tableCell.setLeafTextBlock(textBlock);
        } else {
@@ -181,8 +147,7 @@ public class TableNodeFactory {

    private boolean firstTextBlockIsHeadline(Cell cell) {

-        return cell.getTextBlocks()
-                .get(0).isHeadline();
+        return cell.getTextBlocks().get(0).isHeadline();
    }

 }
@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;

 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
@@ -17,14 +17,14 @@ public class TextBlockFactory {
    long textBlockIdx;


-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<Word> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {

        Integer numberOnPage = context.getAndIncrementTextBlockNumberOnPage(page);
        return buildAtomicTextBlock(sequences, parent, numberOnPage, page);
    }


-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, Integer numberOnPage, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<Word> sequences, SemanticNode parent, Integer numberOnPage, Page page) {

        SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionDto(sequences);
        int offset = stringOffset;
@@ -32,35 +32,34 @@ public class TextBlockFactory {
        long idx = textBlockIdx;
        textBlockIdx++;
        String orientation;
-        int textDirection;
+        int textRotation;
        if (sequences.isEmpty()) {
            orientation = null;
-            textDirection = 0;
+            textRotation = 0;
        } else {
            orientation = sequences.get(0).getDir().toString();
-            textDirection = sequences.get(0).getRotation();
+            textRotation = sequences.get(0).getDir().getRotation();
        }
-        return AtomicTextBlock.fromSearchTextWithTextPosition(searchTextWithTextPositionDto.getSearchText(),
-                searchTextWithTextPositionDto.getLineBreaks(),
-                searchTextWithTextPositionDto.getBoldTextBoundaries(),
-                searchTextWithTextPositionDto.getItalicTextBoundaries(),
-                searchTextWithTextPositionDto.getPositions(),
-                searchTextWithTextPositionDto.getStringIdxToPositionIdx(),
-                idx,
-                parent,
-                numberOnPage,
-                page,
-                offset,
-                orientation,
-                textDirection);
+        var atb = AtomicTextBlock.fromSearchTextWithTextPosition(searchTextWithTextPositionDto.getSearchText(),
+                                                                 searchTextWithTextPositionDto.getLineBreaks(),
+                                                                 searchTextWithTextPositionDto.getBoldTextBoundaries(),
+                                                                 searchTextWithTextPositionDto.getItalicTextBoundaries(),
+                                                                 searchTextWithTextPositionDto.getPositions(),
+                                                                 searchTextWithTextPositionDto.getStringIdxToPositionIdx(),
+                                                                 idx,
+                                                                 parent,
+                                                                 numberOnPage,
+                                                                 page,
+                                                                 offset,
+                                                                 orientation,
+                                                                 textRotation);
+        return atb;
    }


    public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, Page page) {

-        long idx = textBlockIdx;
-        textBlockIdx++;
-        return AtomicTextBlock.empty(idx, stringOffset, page, context.getAndIncrementTextBlockNumberOnPage(page), parent);
+        return emptyTextBlock(parent, context.getAndIncrementTextBlockNumberOnPage(page), page);
    }


@@ -68,7 +67,8 @@ public class TextBlockFactory {

        long idx = textBlockIdx;
        textBlockIdx++;
-        return AtomicTextBlock.empty(idx, stringOffset, page, numberOnPage, parent);
+        var atb = AtomicTextBlock.empty(idx, stringOffset, page, numberOnPage, parent);
+        return atb;
    }

 }
@@ -8,12 +8,11 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -34,10 +33,10 @@ public class GraphicExtractorService {
                                                PDPage pdPage,
                                                int pageNumber,
                                                CleanRulings cleanRulings,
-                                                List<TextPositionSequence> textPositionSequences,
+                                                List<Word> words,
                                                boolean graphicsRaster) {

-        List<Box> characterBBoxes = getCharacterBBoxes(textPositionSequences);
+        List<Box> characterBBoxes = getCharacterBBoxes(words);
        List<Box> classifiedRulingsBoxes = getLineBBoxesOfAllClassifiedRulings(cleanRulings);

        GraphicBBDetector graphicBBDetector = new GraphicBBDetector(pdPage, true);
@@ -64,13 +63,10 @@ public class GraphicExtractorService {
    }


-    private List<Box> getCharacterBBoxes(List<TextPositionSequence> textPositionSequences) {
+    private List<Box> getCharacterBBoxes(List<Word> words) {

-        return textPositionSequences.stream()
-                .map(pos -> pos.getTextPositions()
-                        .stream()
-                        .map(tp -> SearchTextWithTextPositionFactory.mapRedTextPositionToInitialUserSpace(tp, pos))
-                        .collect(RectangleTransformations.collectBBox()))
+        return words.stream()
+                .map(BoundingBox::getBBoxPdf)
                .map(Box::new)
                .collect(Collectors.toList());
    }
@@ -41,7 +41,9 @@ public class DocumentGraphMapper {
        DocumentTree documentTree = new DocumentTree(document);
        Context context = new Context(documentData, documentTree);

-        context.pages.addAll(Arrays.stream(documentData.getDocumentPages()).map(DocumentGraphMapper::buildPage).toList());
+        context.pages.addAll(Arrays.stream(documentData.getDocumentPages())
+                                     .map(DocumentGraphMapper::buildPage)
+                                     .toList());

        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));

@@ -59,7 +61,9 @@ public class DocumentGraphMapper {
        List<DocumentTree.Entry> newEntries = new LinkedList<>();
        for (DocumentStructure.EntryData entryData : entries) {

-            List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();
+            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
+                    .map(pageNumber -> getPage(pageNumber, context))
+                    .toList();

            SemanticNode node = switch (entryData.getType()) {
                case SECTION -> buildSection(context);
@@ -77,16 +81,18 @@ public class DocumentGraphMapper {
            if (entryData.getAtomicBlockIds().length > 0) {
                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
                node.setLeafTextBlock(textBlock);
+                switch (entryData.getType()) {
+                    case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
+                    case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
+                    case IMAGE -> pages.forEach(page -> page.getImages().add((Image) node));
+                    default -> textBlock.getAtomicTextBlocks()
+                            .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
+                }
            }
-            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
+            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
+                    .toList();
            node.setTreeId(treeId);

-            switch (entryData.getType()) {
-                case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
-                case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
-                default -> pages.forEach(page -> page.getMainBody().add(node));
-            }
-
            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
        }
        return newEntries;
@@ -142,6 +148,7 @@ public class DocumentGraphMapper {
        return Section.builder().documentTree(context.documentTree).build();
    }

+
    private SuperSection buildSuperSection(Context context) {

        return SuperSection.builder().documentTree(context.documentTree).build();
@@ -166,22 +173,24 @@ public class DocumentGraphMapper {

    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {

-        return Arrays.stream(atomicTextBlockIds).map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)).collect(new TextBlockCollector());
+        return Arrays.stream(atomicTextBlockIds)
+                .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
+                .collect(new TextBlockCollector());
    }


    private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {

        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                parent,
-                getPage(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                                                       context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
+                                                       parent,
+                                                       getPage(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
    }


    private Page buildPage(DocumentPage p) {

-        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
+        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).textBlocksOnPage(new LinkedList<>()).build();
    }


@@ -206,8 +215,10 @@ public class DocumentGraphMapper {

            this.documentTree = documentTree;
            this.pages = new LinkedList<>();
-            this.documentTextDataBlockData = Arrays.stream(documentData.getDocumentTextData()).toList();
-            this.atomicPositionBlockData = Arrays.stream(documentData.getDocumentPositions()).toList();
+            this.documentTextDataBlockData = Arrays.stream(documentData.getDocumentTextData())
+                    .toList();
+            this.atomicPositionBlockData = Arrays.stream(documentData.getDocumentPositions())
+                    .toList();

        }

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.markdown;
+package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.util.ArrayList;
 import java.util.HashSet;
@@ -32,6 +32,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.He
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;

 public class MarkdownMapper extends AbstractNodeVisitor {

@@ -297,12 +298,6 @@ public class MarkdownMapper extends AbstractNodeVisitor {
    }


-    enum FontStyle {
-        REGULAR,
-        BOLD,
-        ITALIC,
-        BOLD_ITALIC;
-    }

    record FontStyleChange(boolean enter, FontStyle style) {

@@ -0,0 +1,84 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
+
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Optional;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+import com.knecon.fforesight.service.viewerdoc.model.Outline;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class OutlineMapper {
+
+    public Outline createOutline(Document document) {
+
+        Outline outline = new Outline();
+        addChildren(document, null, outline);
+        return outline;
+    }
+
+
+    public void addChildren(SemanticNode parentNode, Outline.Entry parentEntry, Outline outline) {
+
+        parentNode.streamChildren()
+                .filter(child -> child instanceof Section || child instanceof SuperSection)
+                .forEach(child -> {
+                    Optional<Headline> headline = findHeadline(child);
+                    if (headline.isPresent()) {
+                        Outline.Entry entry = buildEntry(child.getHeadline());
+                        if (parentEntry != null) {
+                            parentEntry.children().add(entry);
+                        } else {
+                            outline.getEntries().add(entry);
+                        }
+                        addChildren(child, entry, outline);
+                    } else {
+                        addChildren(child, parentEntry, outline);
+                    }
+                });
+    }
+
+
+    private static Optional<Headline> findHeadline(SemanticNode child) {
+
+        return child.streamChildren()
+                .filter(node -> node instanceof Headline)
+                .map(node -> (Headline) node)
+                .findFirst();
+    }
+
+
+    @SneakyThrows
+    private Outline.Entry buildEntry(Headline headline) {
+
+        Map<Page, Rectangle2D> bbox = headline.getBBox();
+        Rectangle2D r = bbox.get(headline.getFirstPage());
+        Point2D.Double position = new Point2D.Double(r.getMinX(), r.getMaxY());
+        PageInformation pageInformation = PageInformation.fromPage(headline.getFirstPage());
+
+        AffineTransform pdfToPage = CoordinateTransforms.calculateInitialUserSpaceCoordsToPageCoords(pageInformation);
+        pdfToPage.transform(position, position);
+
+        AffineTransform mirror = new AffineTransform(1, 0, 0, -1, 0, pageInformation.heightRot());
+        mirror.transform(position, position);
+
+        AffineTransform.getTranslateInstance(0, 5).transform(position, position);
+
+        Outline.JumpAction action = new Outline.JumpAction(headline.getFirstPage().getNumber(), position);
+        return new Outline.Entry(headline.getTextBlock().getSearchText(), action, new LinkedList<>());
+    }
+
+}
@@ -289,7 +289,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
                                                 new int[]{code},
                                                 font,
                                                 fontSize,
-                                                 (int) (fontSize * textMatrix.getScalingFactorX())));
+                                                 (int) (fontSize * textMatrix.getScalingFactorX() * textMatrix.getScalingFactorY())));
            processTextPosition(new TextPosition(pageRotation,
                                                 pageSize.getWidth(),
                                                 pageSize.getHeight(),
@@ -303,7 +303,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
                                                 new int[]{code},
                                                 font,
                                                 fontSize,
-                                                 (int) (fontSize * textMatrix.getScalingFactorX())));
+                                                 (int) (fontSize * textMatrix.getScalingFactorX() * textMatrix.getScalingFactorY())));
        } else {

            processTextPosition(new TextPosition(pageRotation,
@@ -319,7 +319,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
                                                 new int[]{code},
                                                 font,
                                                 fontSize,
-                                                 (int) (fontSize * textMatrix.getScalingFactorX())));
+                                                 (int) (fontSize * textMatrix.getScalingFactorX() * textMatrix.getScalingFactorY())));
        }
    }

@@ -5,6 +5,7 @@ import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;

 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
@@ -39,7 +40,7 @@ import org.apache.pdfbox.text.TextPosition;

 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.Getter;
 import lombok.Setter;
@@ -50,7 +51,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
 public class PDFLinesTextStripper extends PDFTextStripper {

-    private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+    private final static Set<String> DOT_LIKE_CHARACTERS = Set.of(".", "·", "•", "․", "‧", "∙", "⋅", "・", "．", "･", "…", "⸱", "﹒", "ꞏ");
+    private final List<Word> words = new ArrayList<>();
    private final List<Ruling> rulings = new ArrayList<>();
    private final List<Ruling> graphicsPath = new ArrayList<>();
    @Setter
@@ -201,7 +203,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

        try {
            if (stroke && !getGraphicsState().getStrokingColor().isPattern() && isBlack(getGraphicsState().getStrokingColor()) || //
-                    !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && isBlack(getGraphicsState().getNonStrokingColor())) {
+                !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && isBlack(getGraphicsState().getNonStrokingColor())) {
                rulings.addAll(path);
            }
        } catch (UnsupportedOperationException e) {
@@ -244,10 +246,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                direction = textPositions.get(i).getDir();
            }

-            if (!textPositionSequences.isEmpty()) {
-                previous = textPositionSequences.get(textPositionSequences.size() - 1)
+            if (!words.isEmpty()) {
+                previous = words.get(words.size() - 1)
                        .getTextPositions()
-                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
+                        .get(words.get(words.size() - 1).getTextPositions().size() - 1);
            }

            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
@@ -257,7 +259,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

            if (textPositions.get(i).getDir() != direction && startIndex != i) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                startIndex = i;
                direction = textPositions.get(i).getDir();
            }
@@ -266,7 +268,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                    words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
            }
@@ -274,53 +276,97 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                    words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
            }

-            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i)
-                    .getUnicode()
-                    .equals("\t")) && i <= textPositions.size() - 2) {
+            if (i > 0
+                && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))
+                && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {

                    // Remove false sequence ends (whitespaces)
                    if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
                        for (TextPosition t : sublist) {
-                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
+                            words.get(words.size() - 1).add(t);
                        }
                    } else {
-                        textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                        words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                    }
                }
                startIndex = i + 1;
            }
+            if (isDottedLineFollowedByWord(textPositions, i, startIndex)) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                startIndex = i;
+            }
+            if (isWordFollowedByDottedLine(textPositions, i, startIndex)) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i - 2);
+                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                startIndex = i - 2;
+            }
        }

        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1)
-                .getUnicode()
-                .equals("\u00A0") || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
+        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ")
+                                   || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0")
+                                   || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
            sublist = sublist.subList(0, sublist.size() - 1);
        }

        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
                .getUnicode()
                .equals("\t")))) {
-            if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                    .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
+            if (previous != null
+                && sublist.get(0).getYDirAdj() == previous.getYDirAdj()
+                && sublist.get(0).getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
                for (TextPosition t : sublist) {
-                    textPositionSequences.get(textPositionSequences.size() - 1).add(t);
+                    words.get(words.size() - 1).add(t);
                }
            } else {
-                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, isParagraphStart));
+                words.add(new Word(sublist, pageNumber, isParagraphStart));
            }
        }
+
        super.writeString(text);
    }


+    private boolean isWordFollowedByDottedLine(List<TextPosition> textPositions, int i, int startIndex) {
+
+        return i - startIndex >= 4 //
+               && isDot(textPositions, i) //
+               && isDot(textPositions, i - 1) //
+               && isDot(textPositions, i - 2) //
+               && alphanumeric(textPositions, i - 3);
+    }
+
+
+    private static boolean isDottedLineFollowedByWord(List<TextPosition> textPositions, int i, int startIndex) {
+
+        return i - startIndex >= 4 //
+               && alphanumeric(textPositions, i)  //
+               && isDot(textPositions, i - 1) //
+               && isDot(textPositions, i - 2) //
+               && isDot(textPositions, i - 3);
+    }
+
+
+    private static boolean isDot(List<TextPosition> textPositions, int i) {
+
+        return DOT_LIKE_CHARACTERS.contains(textPositions.get(i).getUnicode());
+    }
+
+
+    private static boolean alphanumeric(List<TextPosition> textPositions, int i) {
+
+        return Character.isAlphabetic(textPositions.get(i).getUnicode().charAt(0)) || Character.isDigit(textPositions.get(i).getUnicode().charAt(0));
+    }
+
+
    public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {

        return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
@@ -337,15 +383,16 @@ public class PDFLinesTextStripper extends PDFTextStripper {

    public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {

-        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
+        return previous != null
+               && sublist.get(0).getYDirAdj() == previous.getYDirAdj()
+               && sublist.get(0).getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
    }


    @Override
    public String getText(PDDocument doc) throws IOException {

-        textPositionSequences.clear();
+        words.clear();
        rulings.clear();
        graphicsPath.clear();
        path_x = 0.0f;
@@ -25,10 +25,22 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.regex.Pattern;

-import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.pdfbox.cos.COSDictionary;
@@ -46,6 +58,8 @@ import org.apache.pdfbox.text.TextPositionComparator;

 import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;

+import lombok.Getter;
+
 /**
 * This is just a copy except i only adjusted lines 594-607 cause this is a bug in Pdfbox.
 * see S416.pdf
@@ -194,40 +208,33 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    }


+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {

-    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
-    {
        PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
-        if (this.currentMarkedContents.isEmpty())
-        {
+        if (this.currentMarkedContents.isEmpty()) {
            this.markedContents.add(markedContent);
-        }
-        else
-        {
-            PDMarkedContent currentMarkedContent =
-                    this.currentMarkedContents.peek();
-            if (currentMarkedContent != null)
-            {
+        } else {
+            PDMarkedContent currentMarkedContent = this.currentMarkedContents.peek();
+            if (currentMarkedContent != null) {
                currentMarkedContent.addMarkedContent(markedContent);
            }
        }
        this.currentMarkedContents.push(markedContent);
    }

+
    @Override
-    public void endMarkedContentSequence()
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void endMarkedContentSequence() {
+
+        if (!this.currentMarkedContents.isEmpty()) {
            this.currentMarkedContents.pop();
        }
    }


-    public void xobject(PDXObject xobject)
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void xobject(PDXObject xobject) {
+
+        if (!this.currentMarkedContents.isEmpty()) {
            this.currentMarkedContents.peek().addXObject(xobject);
        }
    }
@@ -313,7 +320,11 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            endBookmarkPageNumber = -1;
        }

-        if (startBookmarkPageNumber == -1 && startBookmark != null && endBookmarkPageNumber == -1 && endBookmark != null && startBookmark.getCOSObject() == endBookmark.getCOSObject()) {
+        if (startBookmarkPageNumber == -1
+            && startBookmark != null
+            && endBookmarkPageNumber == -1
+            && endBookmark != null
+            && startBookmark.getCOSObject() == endBookmark.getCOSObject()) {
            // this is a special case where both the start and end bookmark
            // are the same but point to nothing. In this case
            // we will not extract any text.
@@ -360,7 +371,9 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    @Override
    public void processPage(PDPage page) throws IOException {

-        if (currentPageNo >= startPage && currentPageNo <= endPage && (startBookmarkPageNumber == -1 || currentPageNo >= startBookmarkPageNumber) && (endBookmarkPageNumber == -1 || currentPageNo <= endBookmarkPageNumber)) {
+        if (currentPageNo >= startPage && currentPageNo <= endPage && (startBookmarkPageNumber == -1 || currentPageNo >= startBookmarkPageNumber) && (endBookmarkPageNumber == -1
+                                                                                                                                                      || currentPageNo
+                                                                                                                                                         <= endBookmarkPageNumber)) {
            startPage(page);

            int numberOfArticleSections = 1;
@@ -635,7 +648,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                        var normalized = normalize(line);
 //                        normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent()

-
                        lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine);
                        writeLine(normalized, current.isParagraphStart);
                        line.clear();
@@ -647,8 +659,8 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    }
                    // test if our TextPosition starts after a new word would be expected to start
                    if (expectedStartOfNextWordX != EXPECTED_START_OF_NEXT_WORD_X_RESET_VALUE && expectedStartOfNextWordX < positionX
-                            // only bother adding a word separator if the last character was not a word separator
-                            && (wordSeparator.isEmpty() || //
+                        // only bother adding a word separator if the last character was not a word separator
+                        && (wordSeparator.isEmpty() || //
                            (lastPosition.getTextPosition().getUnicode() != null && !lastPosition.getTextPosition().getUnicode().endsWith(wordSeparator)))) {
                        line.add(LineItem.getWordSeparator());
                    }
@@ -914,8 +926,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    textList.add(text);
                }
            }
-            if (!this.currentMarkedContents.isEmpty())
-            {
+            if (!this.currentMarkedContents.isEmpty()) {
                this.currentMarkedContents.peek().addText(text);
            }
        }
@@ -1711,7 +1722,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
        int numberOfStrings = line.size();
        for (int i = 0; i < numberOfStrings; i++) {
            WordWithTextPositions word = line.get(i);
-            word.getTextPositions().sort(Comparator.comparing(TextPosition::getXDirAdj));
            writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
            if (i < numberOfStrings - 1) {
                writeWordSeparator();
@@ -2102,7 +2112,9 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            return endParagraphWritten;
        }

-        public void setEndParagraphWritten(){
+
+        public void setEndParagraphWritten() {
+
            endParagraphWritten = true;
        }

@@ -2145,7 +2157,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            this.isHangingIndent = true;
        }

-
    }

 }
@@ -10,7 +10,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.He
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.OutlineMapper;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutGrid;
+import com.knecon.fforesight.service.viewerdoc.model.Outline;
 import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;

 import io.micrometer.observation.annotation.Observed;
@@ -29,16 +31,18 @@ public class LayoutGridService {

    @SneakyThrows
    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
-    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue, boolean writeVisualLayoutParsingGrid) {
+    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {

        LayoutGrid layoutGrid = createLayoutGrid(document);
+        Outline outline = OutlineMapper.createOutline(document);
        layoutGrid.setVisibleByDefault(layerVisibilityDefaultValue);
-//        Visualizations visualLayoutGrid = this.addLayoutGrid(document, layerVisibilityDefaultValue, true);
-        if (document.getLayoutDebugLayer().isActive()) {
-            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid, document.getLayoutDebugLayer()));
-        } else {
-            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid));

+        document.getLayoutDebugLayer().addSentenceVisualization(document.getTextBlock());
+
+        if (document.getLayoutDebugLayer().isActive()) {
+            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid, document.getLayoutDebugLayer()), outline);
+        } else {
+            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid), outline);
        }
    }

@@ -32,7 +32,6 @@ public class CoordinateTransforms {
    }


-
    @SneakyThrows
    public AffineTransform calculateInitialUserSpaceCoordsToImageCoords(PageInformation pageInformation, double scalingFactor) {

@@ -40,6 +39,19 @@ public class CoordinateTransforms {
    }


+    public AffineTransform calculatePageCoordsToInitialUserSpaceCoords(PageInformation pageInformation) {
+
+        return calculateImageCoordsToInitialUserSpaceCoords(pageInformation, 1);
+    }
+
+
+    @SneakyThrows
+    public AffineTransform calculateInitialUserSpaceCoordsToPageCoords(PageInformation pageInformation) {
+
+        return calculatePageCoordsToInitialUserSpaceCoords(pageInformation).createInverse();
+    }
+
+
    public double calculateScalingFactor(PageInformation pageInformation, double imageWidth) {

        // PDFBox always returns page height and width based on rotation
@@ -1,5 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;

+import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.FOOTER;
+import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.HEADER;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -9,6 +12,7 @@ import java.util.stream.Collectors;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;

 import lombok.experimental.UtilityClass;
@@ -26,35 +30,60 @@ public class HeaderFooterDetection {

    public boolean isLikelyFooter(TextPageBlock textPageBlock, ClassificationDocument document, ClassificationPage classificationPage) {

-        int numberOfPages = document.getPages().size();
-        if (numberOfPages < 3) {
-            // If the document has 1 or 2 pages this may lead to more false positives than actual findings.
-            return false;
-        }
-
-        int window = Math.min(numberOfPages, 8);
-
-        List<ClassificationPage> nearestPages = findNearestPages(classificationPage, document.getPages(), window);
-        List<List<AbstractPageBlock>> footerCandidates = getFooterCandidates(nearestPages);
-
-        return detectHeadersOrFootersByPageAssociation(textPageBlock.getText(), footerCandidates, window, footerWeights);
+        return isLikelyHeaderFooter(textPageBlock, document, classificationPage, FOOTER);
    }


    public boolean isLikelyHeader(TextPageBlock textPageBlock, ClassificationDocument document, ClassificationPage classificationPage) {

+        return isLikelyHeaderFooter(textPageBlock, document, classificationPage, HEADER);
+    }
+
+
+    private boolean isLikelyHeaderFooter(TextPageBlock textPageBlock, ClassificationDocument document, ClassificationPage classificationPage, PageBlockType type) {
+
        int numberOfPages = document.getPages().size();
        if (numberOfPages < 3) {
            // If the document has 1 or 2 pages this may lead to more false positives than actual findings.
            return false;
        }

+        List<TextPageBlock> textPageBlocks = classificationPage.getTextBlocks()
+                .stream()
+                .filter(TextPageBlock.class::isInstance)
+                .map(TextPageBlock.class::cast)
+                .collect(Collectors.toList());
+
+        if (textPageBlocks.isEmpty()) {
+            return false;
+        }
+
+        List<TextPageBlock> selectedBlocks;
+        if (type == HEADER) {
+            selectedBlocks = textPageBlocks.subList(0, Math.min(3, textPageBlocks.size()));
+        } else { //FOOTER
+            selectedBlocks = textPageBlocks.subList(Math.max(0, textPageBlocks.size() - 3), textPageBlocks.size());
+        }
+
+        if (!selectedBlocks.contains(textPageBlock)) {
+            // The textPageBlock is not among the selected blocks on its page
+            return false;
+        }
+
        int window = Math.min(numberOfPages, 8);
-
        List<ClassificationPage> nearestPages = findNearestPages(classificationPage, document.getPages(), window);
-        List<List<AbstractPageBlock>> headerCandidates = getHeaderCandidates(nearestPages);

-        return detectHeadersOrFootersByPageAssociation(textPageBlock.getText(), headerCandidates, window, headerWeights);
+        List<List<AbstractPageBlock>> candidates;
+        double[] weights;
+        if (type == HEADER) {
+            candidates = getHeaderCandidates(nearestPages);
+            weights = headerWeights;
+        } else { //FOOTER
+            candidates = getFooterCandidates(nearestPages);
+            weights = footerWeights;
+        }
+
+        return detectHeadersOrFootersByPageAssociation(textPageBlock.getText(), candidates, window, weights);
    }


@@ -96,7 +125,7 @@ public class HeaderFooterDetection {
                        continue;
                    }

-                    int distance = calculateHammingDistanceWithPreprocessing(testString, paddedString);
+                    int distance = StringDistances.hamming(testString, paddedString);
                    double normalizedScore = 1 - (double) distance / Math.max(testString.length(), paddedString.length());
                    score += normalizedScore * (j < weights.length ? weights[j] : 1);
                }
@@ -180,44 +209,4 @@ public class HeaderFooterDetection {
        return headerCandidates;
    }

-
-    /**
-     * Calculate the Hamming distance between two strings after preprocessing to make them the same length
-     * and replacing all digits with a special character '@' since they are a common occurrence in headers/footers.
-     *
-     * @param firstCandidate  First string
-     * @param secondCandidate Second string
-     * @return The Hamming distance between the two preprocessed strings.
-     */
-    private int calculateHammingDistanceWithPreprocessing(String firstCandidate, String secondCandidate) {
-
-        int maxLength = Math.max(firstCandidate.length(), secondCandidate.length());
-
-        String cleanFirstCandidate = padString(firstCandidate, maxLength, '\0').replaceAll("\\d", "@");
-        String cleanSecondCandidate = padString(secondCandidate, maxLength, '\0').replaceAll("\\d", "@");
-
-        int distance = 0;
-        for (int i = 0; i < maxLength; i++) {
-            if (cleanFirstCandidate.charAt(i) != cleanSecondCandidate.charAt(i)) {
-                distance++;
-            }
-        }
-        return distance;
-    }
-
-
-    private String padString(String input, int length, char padChar) {
-
-        if (input.length() >= length) {
-            return input;
-        }
-
-        StringBuilder sb = new StringBuilder(input);
-
-        while (sb.length() < length) {
-            sb.append(padChar);
-        }
-        return sb.toString();
-    }
-
 }
@@ -13,7 +13,7 @@ import org.apache.pdfbox.text.TextPosition;

 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@@ -48,7 +48,7 @@ public class MarkedContentUtils {

        return markedContentByYPosition.values()
                .stream()
-                .map(textPositions -> new TextPositionSequence(textPositions, 0, true).getBBoxInitialUserSpace())
+                .map(textPositions -> new Word(textPositions, 0, true).getBBoxPdf())
                .map(t -> new Rectangle2D.Double(t.getX(), t.getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight())))
                .collect(Collectors.toList());
    }
@@ -89,8 +89,8 @@ public class MarkedContentUtils {
                    .filter(content -> content instanceof TextPosition)
                    .map(content -> (TextPosition) content)
                    .filter(content -> !content.getUnicode().equals(" "))
-                    .map(textPositions -> new TextPositionSequence(List.of(textPositions), 0, true))
-                    .map(BoundingBox::getBBoxInitialUserSpace)
+                    .map(textPositions -> new Word(List.of(textPositions), 0, true))
+                    .map(BoundingBox::getBBoxPdf)
                    .collect(Collectors.toList());
        }

@@ -5,14 +5,22 @@ import java.awt.geom.Rectangle2D;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;

+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+
 public record PageInformation(Rectangle2D mediabox, int number, int rotationDegrees) {

    public static PageInformation fromPDPage(int pageNum, PDPage page) {

        PDRectangle mediaBox = page.getMediaBox();
        return new PageInformation(new Rectangle2D.Double(mediaBox.getLowerLeftX(), mediaBox.getLowerLeftY(), mediaBox.getWidth(), mediaBox.getHeight()),
-                pageNum,
-                page.getRotation());
+                                   pageNum,
+                                   page.getRotation());
+    }
+
+
+    public static PageInformation fromPage(Page page) {
+
+        return new PageInformation(new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight()), page.getNumber(), page.getRotation());
    }


@@ -108,7 +108,7 @@ public final class PositionUtils {
    }


-    public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextPageBlock textBlock, Float documentMostPopularWordHeight) {
+    public double getHeightDifferenceBetweenChunkWordAndDocumentWord(TextPageBlock textBlock, Double documentMostPopularWordHeight) {

        return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
    }
@@ -116,7 +116,7 @@ public final class PositionUtils {

    public double getApproxLineCount(TextPageBlock textBlock) {

-        return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
+        return textBlock.getBBoxDirAdj().getHeight() / textBlock.getMostPopularWordHeight();
    }

 }
@@ -28,7 +28,7 @@ public class SpreadsheetFinder {
        Map<Point2D, Point2D> edgesV = new HashMap<>();

        for (Cell cell : cells) {
-            for (Point2D pt : getPoints(cell.getBBoxInitialUserSpace())) {
+            for (Point2D pt : getPoints(cell.getBBoxPdf())) {
                if (pointSet.contains(pt)) { // shared vertex, remove it
                    pointSet.remove(pt);
                } else {
@@ -0,0 +1,49 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class StringDistances {
+
+
+    /**
+     * Calculate the Hamming distance between two strings after preprocessing to make them the same length
+     * and replacing all digits with a special character '@' since they are a common occurrence in headers/footers.
+     *
+     * @param s1  First string
+     * @param s2 Second string
+     * @return The Hamming distance between the two preprocessed strings.
+     */
+    public int hamming(String s1, String s2) {
+
+        int maxLength = Math.max(s1.length(), s2.length());
+
+        String cleanFirstCandidate = padString(s1, maxLength, '\0').replaceAll("\\d", "@");
+        String cleanSecondCandidate = padString(s2, maxLength, '\0').replaceAll("\\d", "@");
+
+        int distance = 0;
+        for (int i = 0; i < maxLength; i++) {
+            if (cleanFirstCandidate.charAt(i) != cleanSecondCandidate.charAt(i)) {
+                distance++;
+            }
+        }
+        return distance;
+    }
+
+
+    private String padString(String input, int length, char padChar) {
+
+        if (input.length() >= length) {
+            return input;
+        }
+
+        StringBuilder sb = new StringBuilder(input);
+
+        while (sb.length() < length) {
+            sb.append(padChar);
+        }
+        return sb.toString();
+    }
+
+
+}
@@ -1,9 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;

 import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.stream.Stream;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -22,29 +23,77 @@ public class TableMergingUtility {
        List<TablePageBlock> consecutiveTables = pageBlocks.stream()
                .map(abstractPageBlock -> (TablePageBlock) abstractPageBlock)
                .filter(tablePageBlock -> !tablePageBlock.equals(originalTablePageBlock))
+                .sorted(Comparator.comparingInt(TablePageBlock::getPage).thenComparing(TablePageBlock::getY).thenComparing(TablePageBlock::getX))
                .toList();
+
        assert consecutiveTables.size() == pageBlocks.size() - 1;
+        var currentTable = originalTablePageBlock;
+        int currentTableIndex = 0;

        List<TablePageBlock> consecutiveTablesWithSameColCountAndHeaders = new LinkedList<>();
-        for (TablePageBlock consecutiveTable : consecutiveTables) {
-            if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() && !hasTableHeader(consecutiveTable) && outerBoundaryAlignsX(originalTablePageBlock,
-                    consecutiveTable)) {
+        consecutiveTablesWithSameColCountAndHeaders.add(originalTablePageBlock);
+        for (int i = 0; i < consecutiveTables.size(); i++) {
+            TablePageBlock consecutiveTable = consecutiveTables.get(i);
+
+            if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() //
+                && getHeaders(consecutiveTable).isEmpty() //
+                && outerBoundaryAlignsX(originalTablePageBlock, consecutiveTable) //
+                && consecutiveOrSamePage(currentTable, consecutiveTable) //
+                && !tableBetween(currentTable, consecutiveTable, findTablesBetween(consecutiveTables, currentTableIndex, i))) {
+
+                currentTable = consecutiveTable;
+                currentTableIndex = i;
                consecutiveTablesWithSameColCountAndHeaders.add(consecutiveTable);
            }
        }
-        return Stream.concat(Stream.of(originalTablePageBlock), consecutiveTablesWithSameColCountAndHeaders.stream()).toList();
+        return consecutiveTablesWithSameColCountAndHeaders;
+    }
+
+
+    private static List<TablePageBlock> findTablesBetween(List<TablePageBlock> consecutiveTables, int currentTableIndex, int i) {
+
+        if (currentTableIndex + 1 == consecutiveTables.size() || currentTableIndex + 1 >= i) {
+            return Collections.emptyList();
+        }
+        return consecutiveTables.subList(currentTableIndex + 1, i);
+    }
+
+
+    private static boolean consecutiveOrSamePage(TablePageBlock currentTable, TablePageBlock consecutiveTable) {
+
+        return currentTable.getPage() == consecutiveTable.getPage() || currentTable.getPage() + 1 == consecutiveTable.getPage();
+    }
+
+
+    private static boolean tableBetween(TablePageBlock currentTable, TablePageBlock consecutiveTable, List<TablePageBlock> tablesBetween) {
+
+        if (tablesBetween.isEmpty()) {
+            return false;
+        }
+        // assumes the tables are on the same page or on consecutive pages, all tables on pages in between are ignored.
+        return tablesBetween.stream()
+                       .filter(tableBetween -> tableBetween.getPage() == currentTable.getPage())
+                       .anyMatch(tableBetween -> tableBetween.isBelow(currentTable)) //
+               || tablesBetween.stream()
+                       .filter(tableBetween -> tableBetween.getPage() == consecutiveTable.getPage())
+                       .anyMatch(tableBetween -> tableBetween.isAbove(consecutiveTable));
    }


    private static boolean outerBoundaryAlignsX(TablePageBlock originalTablePageBlock, TablePageBlock consecutiveTable) {

-        return Math.abs(consecutiveTable.getMinX() - originalTablePageBlock.getMinX()) < TABLE_ALIGNMENT_THRESHOLD && Math.abs(consecutiveTable.getMaxX() - originalTablePageBlock.getMaxX()) < TABLE_ALIGNMENT_THRESHOLD;
+        return Math.abs(consecutiveTable.getMinX() - originalTablePageBlock.getMinX()) < TABLE_ALIGNMENT_THRESHOLD
+               && Math.abs(consecutiveTable.getMaxX() - originalTablePageBlock.getMaxX()) < TABLE_ALIGNMENT_THRESHOLD;
    }


-    private boolean hasTableHeader(TablePageBlock table) {
+    private List<Cell> getHeaders(TablePageBlock table) {

-        return table.getRows().stream().flatMap(Collection::stream).anyMatch(Cell::isHeaderCell);
+        return table.getRows()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(Cell::isHeaderCell)
+                .toList();
    }

 }
@@ -1,31 +1,47 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;

+import java.util.regex.Pattern;
+
 import lombok.experimental.UtilityClass;

@UtilityClass
 public final class TextNormalizationUtilities {

-    /**
-     * Revert hyphenation due to line breaks.
-     *
-     * @param text Text to be processed.
-     * @return Text without line-break hyphenation.
-     */
-    public static String removeHyphenLineBreaks(String text) {
+    public static final Pattern hyphenLineBreaks = Pattern.compile("[-~‐‒⁻−﹣゠⁓‑\\u00AD][\\r\\n]+");
+    public static final Pattern linebreaks = Pattern.compile("[\\r\\n]+");
+    public static final Pattern doubleWhitespaces = Pattern.compile("\\s{2,}");
+    public static final Pattern WHITESPACE_REMOVAL = Pattern.compile("\\s+");

-        return text.replaceAll("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R", "$1");
+
+    public String cleanString(String value) {
+
+        String noHyphenLinebreaks = removeHyphenLinebreaks(value);
+        String noLinebreaks = removeLinebreaks(noHyphenLinebreaks);
+        return removeMultipleWhitespaces(noLinebreaks);
    }


-    public static String removeLineBreaks(String text) {
+    public String removeHyphenLinebreaks(String value) {

-        return text.replaceAll("\n", " ");
+        return hyphenLineBreaks.matcher(value).replaceAll("");
    }


-    public static String removeRepeatingWhitespaces(String text) {
+    private String removeMultipleWhitespaces(String value) {

-        return text.replaceAll(" {2}", " ");
+        return doubleWhitespaces.matcher(value).replaceAll(" ");
+    }
+
+
+    private String removeLinebreaks(String value) {
+
+        return linebreaks.matcher(value).replaceAll(" ");
+    }
+
+
+    public String removeAllWhitespaces(String value) {
+
+        return WHITESPACE_REMOVAL.matcher(value).replaceAll("");
    }

 }
@@ -1,30 +1,155 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;

+import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;

+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

+import lombok.experimental.UtilityClass;
+
+@UtilityClass
 public class TextPositionOperations {

-    private static final TextPositionSequenceComparator comparator = new TextPositionSequenceComparator();
+    public static final double ANGLE_TOLERANCE = Math.PI / 35;
+    public static final AngleFilter ANGLE_FILTER = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
+    public static final double MAX_LINE_HEIGHT_FACTOR = 0.66; // multiplied with max word height
+    public static final double MAX_WORD_DISTANCE_FACTOR = 3.5; // multiplied with max word width
+
+    private static final double Y_THRESHOLD = 5;
+    private static final double X_THRESHOLD = 3;
+    private static final Comparator<TextBoundingBox> COMPARATOR_DIR_ADJ = //
+            Comparator.comparing(TextBoundingBox::getDir)
+                    .thenComparing(TextBoundingBox::getMaxYDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, Y_THRESHOLD))
+                    .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, X_THRESHOLD));


-    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<TextPageBlock> textBlocks) {
+    public List<Word> mergeAndSort(List<TextPageBlock> textBlocks) {

-        var sequence = textBlocks.stream().flatMap(tb -> tb.getSequences().stream()).collect(Collectors.toList());
-
-        // because the TextPositionSequenceComparator is not transitive, but
-        // JDK7+ enforces transitivity on comparators, we need to use
-        // a custom quicksort implementation (which is slower, unfortunately).
-        QuickSort.sort(sequence, comparator);
-        return sequence;
+        var sequences = textBlocks.stream()
+                .flatMap(tb -> tb.getWords()
+                        .stream())
+                .collect(Collectors.toSet());
+        return sortUsingLineDetection(sequences);
    }

-    public static List<TextPositionSequence> mergeTextPositionSequence(List<TextPageBlock> textBlocks) {

-       return textBlocks.stream().flatMap(tb -> tb.getSequences().stream()).collect(Collectors.toList());
+    public List<Word> sort(List<Word> sequences) {
+
+        return sortUsingLineDetection(new HashSet<>(sequences));
+    }
+
+
+    private List<Word> sortUsingLineDetection(Set<Word> sequences) {
+
+        return sortLines(groupByLine(sequences));
+    }
+
+
+    public List<Word> sortLines(Collection<Set<Word>> lines) {
+
+        List<List<Word>> lineBlocks = new ArrayList<>();
+        for (Set<Word> line : lines) {
+            List<Word> sortedLine = sortByXDirAdj(line);
+            if (!sortedLine.isEmpty()) {
+                lineBlocks.add(sortedLine);
+            }
+        }
+        // need to use old sorting, since COMPARATOR_DIR_ADJ is not transitive
+        QuickSort.sort(lineBlocks, Comparator.comparing(line -> line.get(0), COMPARATOR_DIR_ADJ));
+
+        List<Word> list = new ArrayList<>();
+        for (List<Word> words : lineBlocks) {
+            list.addAll(words);
+        }
+        return list;
+    }
+
+
+    private List<Word> sortByXDirAdj(Set<Word> line) {
+
+        return line.stream()
+                .sorted(Comparator.comparing(Word::getXDirAdj))
+                .toList();
+    }
+
+
+    public Collection<Set<Word>> groupByLine(Set<Word> sequences) {
+
+        double maxLineDistance = sequences.stream()
+                                         .map(Word::getBBoxDirAdj)
+                                         .mapToDouble(RectangularShape::getHeight).average().orElse(10) * MAX_LINE_HEIGHT_FACTOR;
+        double maxXGap = sequences.stream()
+                                 .map(Word::getBBoxDirAdj)
+                                 .mapToDouble(RectangularShape::getWidth).average().orElse(75) * MAX_WORD_DISTANCE_FACTOR;
+
+        UnionFind<Word> unionFind = new UnionFind<>(sequences);
+
+        for (Word sequence : sequences) {
+            for (Word sequence2 : sequences) {
+
+                if (sequence.equals(sequence2)) { // || unionFind.inSameSet(sequence, sequence2)) doing this is actually slower than not doing it
+                    continue;
+                }
+
+                double angle = computeAngle(sequence.getBBoxDirAdj(), sequence2.getBBoxDirAdj());
+
+                double normalizedVerticalDistance = Math.abs(sequence.getBBoxDirAdj().getCenterY() - sequence2.getBBoxDirAdj().getCenterY()) / maxLineDistance;
+                double normalizedHorizontalDistance = Math.abs(sequence.getBBoxDirAdj().getCenterX() - sequence2.getBBoxDirAdj().getCenterX()) / maxXGap;
+
+                if (sequence.getDir() != sequence2.getDir()) {
+                    continue;
+                }
+                if (Math.abs(sequence.getFontSize() - sequence2.getFontSize()) > 0.5 * Math.max(sequence.getFontSize(), sequence2.getFontSize())) {
+                    continue;
+                }
+                if (Math.pow(normalizedVerticalDistance, 2) + Math.pow(normalizedHorizontalDistance, 2) > 1) {
+                    continue;
+                }
+                if (!ANGLE_FILTER.matches(angle)) {
+                    continue;
+                }
+
+                unionFind.union(sequence, sequence2);
+            }
+        }
+
+        return unionFind.getGroups();
+    }
+
+
+    public double computeAngle(Rectangle2D rect1, Rectangle2D rect2) {
+
+        double rect1CentroidX = rect1.getCenterX();
+        double rect1CentroidY = rect1.getCenterY();
+        double rect2CentroidX = rect2.getCenterX();
+        double rect2CentroidY = rect2.getCenterY();
+
+        double deltaX = rect2CentroidX - rect1CentroidX;
+        double deltaY = rect2CentroidY - rect1CentroidY;
+
+        return FastAtan2.fastAtan2(deltaY, deltaX);
+    }
+
+
+    public List<Word> merge(List<TextPageBlock> textBlocks) {
+
+        return textBlocks.stream()
+                .map(TextPageBlock::getWords)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toList());
    }

 }
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Kilian Schüttler	25929cfe2f	Merge branch 'RM-231-bp-bp' into 'release/0.159.x' RM-231: missing whitespace in name See merge request fforesight/layout-parser!265	2025-01-14 13:05:03 +01:00
Kilian Schuettler	b447788fcb	RM-231: missing whitespace in name	2025-01-14 13:00:00 +01:00
Kilian Schüttler	65671d6f25	Merge branch 'RED-10714-bp-bp' into 'release/0.159.x' RED-10714: fix IndexOutOfBoundsException See merge request fforesight/layout-parser!260	2025-01-10 12:33:14 +01:00
Kilian Schuettler	3ef0d9eccd	RED-10714: fix IndexOutOfBoundsException	2025-01-10 12:10:35 +01:00
Kilian Schüttler	64141fc7a2	Merge branch 'RED-10127-bp' into 'release/0.159.x' RED-10127: Paragraphs with multiple table, appendix, figure can't be headlines See merge request fforesight/layout-parser!258	2024-12-06 14:56:05 +01:00
Kilian Schuettler	fcbc005102	RED-10127: Paragraphs with multiple table, appendix, figure can't be headlines	2024-12-06 14:49:35 +01:00
Kilian Schuettler	2fdc53429c	fix accidental push to main	2024-11-28 12:57:13 +01:00
Kilian Schüttler	3b30732352	RED-9139: more robust TOC detection (cherry picked from commit `7ee1f9e360`)	2024-11-28 12:52:55 +01:00
Kilian Schüttler	e01c0a8d3b	Merge branch 'RED-10270-bp' into 'release/0.159.x' RED-10270: fix NumberFormatException See merge request fforesight/layout-parser!250	2024-10-24 17:14:51 +02:00
Kilian Schüttler	5ef5d5509b	RED-10270: fix NumberFormatException	2024-10-24 17:14:51 +02:00
Kilian Schüttler	ab70536d06	Merge branch 'RED-10204' into 'release/0.159.x' RED-10204: backport of NPE hotfix and rename TextPositionSequence to Word See merge request fforesight/layout-parser!247	2024-10-24 10:05:00 +02:00
Kilian Schüttler	5e091402c7	RED-10204: backport of NPE hotfix and rename TextPositionSequence to Word	2024-10-24 10:04:59 +02:00
Kilian Schüttler	03f5acd417	Merge branch 'feature/RED-10127-bp' into 'release/0.159.x' RED-10127: add more units See merge request fforesight/layout-parser!241	2024-10-15 09:57:11 +02:00
Kilian Schuettler	8ca41cf340	RED-10127: add more units	2024-10-15 09:47:07 +02:00
Kilian Schüttler	cee6c74d73	Merge branch 'feature/RED-10127-bp' into 'release/0.159.x' RED-10127: improve list classification See merge request fforesight/layout-parser!239	2024-10-14 17:31:24 +02:00
Kilian Schuettler	d8394d9a78	RED-10127: improve list classification * add one more format to list identification * add 'ppb' to known units * special case for headlines continuing with 14C after the identifier (quite often in some specific files)	2024-10-14 17:22:19 +02:00
Kilian Schüttler	d3c4413ece	Merge branch 'feature/RED-10127-bp' into 'release/0.159.x' RED-10127: add list classification See merge request fforesight/layout-parser!238	2024-10-10 10:50:18 +02:00
Kilian Schüttler	d614aed96a	RED-10127: add list classification	2024-10-10 10:50:17 +02:00
Kilian Schüttler	63953ecf2d	Merge branch 'feature/RED-10127-bp' into 'release/0.159.x' RED-10127: improve headline detection See merge request fforesight/layout-parser!236	2024-10-09 09:56:05 +02:00
Kilian Schüttler	8c28a46817	RED-10127: improve headline detection	2024-10-09 09:56:04 +02:00
Maverick Studer	072ad3bf23	Merge branch 'RED-10126-bp' into 'release/0.159.x' RM-187: Footers are recognized in the middle of the page See merge request fforesight/layout-parser!234	2024-10-08 14:27:55 +02:00
Maverick Studer	8a11d838b9	RM-187: Footers are recognized in the middle of the page	2024-10-08 14:27:55 +02:00
Dominique Eifländer	ed37b4bedf	Merge branch 'RED-9975-4.2' into 'release/0.159.x' RED-9975: Fixed missing section numbers in layout grid See merge request fforesight/layout-parser!229	2024-09-18 11:26:10 +02:00
Dominique Eifländer	dda5a2c719	RED-9975: Fixed missing section numbers in layout grid	2024-09-18 11:20:15 +02:00
Dominique Eifländer	0f641670f7	Merge branch 'RED-9974-4.2' into 'release/0.159.x' Red 9974 4.2 See merge request fforesight/layout-parser!228	2024-09-16 14:06:40 +02:00
Dominique Eifländer	b08c102f76	RED-9974: Disabled failing test because of different header/footers	2024-09-16 13:32:44 +02:00
Dominique Eifländer	6acc85266c	RED-9974: Ignore enoughChars when section identifierer regex matches for documine old	2024-09-16 12:16:11 +02:00
Dominique Eifländer	a4d6d2326e	RED-9974: Do not rewrite outline as pdftron crashes in some cases	2024-09-16 10:50:24 +02:00
Dominique Eifländer	a337fdf684	RED-9974: Ignore pmd errors that only occur on build server	2024-09-16 10:18:27 +02:00
Kilian Schuettler	95e6fdecd7	RED-9974: wip	2024-09-16 09:46:41 +02:00
Kilian Schuettler	1337c56591	RED-9974: wip	2024-09-16 09:46:31 +02:00
Kilian Schuettler	31bf4ba8c8	hotfix: viewerDocService doesn't remove existing marked content	2024-09-16 09:46:16 +02:00
Kilian Schüttler	f034c5bfa0	Merge branch 'RED-9975-bp' into 'release/0.159.x' RED-9975: improve SuperSection handling See merge request fforesight/layout-parser!224	2024-09-11 13:38:04 +02:00
Kilian Schüttler	41ba531734	RED-9975: improve SuperSection handling	2024-09-11 13:38:04 +02:00
Dominique Eifländer	c392813402	Merge branch 'RED-9976-4.2' into 'release/0.159.x' RED-9976: Removed sorting that scrambles text in PDFTextStripper See merge request fforesight/layout-parser!221	2024-09-10 13:02:22 +02:00
Dominique Eifländer	4a624f9642	RED-9976: Removed sorting that scrambles text in PDFTextStripper	2024-09-10 12:48:28 +02:00
Kilian Schüttler	f6c60aa5eb	Merge branch 'hotfix-bp' into 'release/0.159.x' hotfix: unmerge super large tables See merge request fforesight/layout-parser!219	2024-09-05 15:05:11 +02:00
Kilian Schuettler	90a1187921	hotfix: unmerge super large tables	2024-09-05 14:50:35 +02:00
Kilian Schuettler	09c18c110a	hotfix: unmerge super large tables	2024-09-05 14:26:45 +02:00
Kilian Schüttler	9012162542	Merge branch 'hotfix-bp' into 'release/0.159.x' hotfix: add Java advanced imaging See merge request fforesight/layout-parser!216	2024-09-04 15:44:02 +02:00
Kilian Schuettler	49604cd96e	hotfix: add Java advanced imaging	2024-09-04 15:19:43 +02:00
Kilian Schüttler	943a6b6536	Merge branch 'RED-9964-bp' into 'release/0.159.x' RED-9964: fix errors with images See merge request fforesight/layout-parser!213	2024-09-04 09:17:19 +02:00
Kilian Schuettler	302d8b884f	RED-9964: fix errors with images	2024-09-03 16:38:17 +02:00
Dominique Eifländer	a50b047cbb	Merge branch 'RED-9988-4.2' into 'release/0.159.x' RED-9988: Fixed NPE when image representation is not present See merge request fforesight/layout-parser!209	2024-09-02 09:26:16 +02:00
Dominique Eifländer	8de9d8309f	RED-9988: Fixed NPE when image representation is not present	2024-09-02 09:18:38 +02:00
Kilian Schüttler	3b12242355	Merge branch 'RED-9975-bp' into 'release/0.159.x' Red 9975: fix outline detection See merge request fforesight/layout-parser!208	2024-08-30 17:48:02 +02:00
Kilian Schüttler	e8605f4956	Red 9975: fix outline detection	2024-08-30 17:48:02 +02:00
Kilian Schüttler	f4a5b5fcbf	Merge branch 'RED-9975-bp' into 'release/0.159.x' Red 9975: add outline debug layer See merge request fforesight/layout-parser!207	2024-08-30 14:18:09 +02:00
Kilian Schüttler	8496b48cde	Red 9975: add outline debug layer	2024-08-30 14:18:09 +02:00
Kilian Schüttler	de266dcfe5	Merge branch 'RED-9964' into 'release/0.159.x' Red 9964: don't merge tables on non-consecutive pages or with tables in between See merge request fforesight/layout-parser!204	2024-08-30 14:00:50 +02:00
Kilian Schüttler	10e525f0de	Red 9964: don't merge tables on non-consecutive pages or with tables in between	2024-08-30 14:00:50 +02:00
Dominique Eifländer	e0e5e35b30	Merge branch 'RED-9974-4.2' into 'release/0.159.x' RED-9974: Improved headline detection for documine old See merge request fforesight/layout-parser!203	2024-08-30 10:52:31 +02:00
Dominique Eifländer	e1d8d1ea3b	RED-9974: Improved headline detection for documine old	2024-08-30 10:35:24 +02:00
Kilian Schüttler	1546c05dd8	Merge branch 'RED-9975-bp' into 'release/0.159.x' activate outline detection See merge request fforesight/layout-parser!200	2024-08-29 14:26:14 +02:00
Kilian Schuettler	7c88c30ca7	RED-9975: activate outline detection	2024-08-29 14:17:20 +02:00
Kilian Schüttler	50427d08dc	Merge branch 'RED-9975-bp' into 'release/0.159.x' RED-9975: activate outline detection See merge request fforesight/layout-parser!199	2024-08-29 12:43:14 +02:00
Kilian Schuettler	338c6c5dd0	RED-9975: activate outline detection	2024-08-29 12:27:20 +02:00
Dominique Eifländer	3eb97d614f	Merge branch 'RED-9760-NPE' into 'main' RED-9760: Fixed nullpointer in TextPageBlock See merge request fforesight/layout-parser!194	2024-08-13 13:24:48 +02:00
Dominique Eifländer	81469413b0	RED-9760: Fixed nullpointer in TextPageBlock	2024-08-13 13:18:50 +02:00
Dominique Eifländer	2993676a6f	Merge branch 'RED-9670' into 'main' RED-9760: change compareDouble to something sensible See merge request fforesight/layout-parser!193	2024-08-12 16:02:51 +02:00
Kilian Schüttler	8e115dcd8a	RED-9760: change compareDouble to something sensible	2024-08-12 16:02:50 +02:00
Dominique Eifländer	173911b840	Merge branch 'hotfix-reading-order' into 'main' hotfix: threshold adjustements See merge request fforesight/layout-parser!192	2024-08-12 14:59:20 +02:00
Kilian Schuettler	b0ae00aa02	hotfix: threshold adjustements	2024-08-12 14:52:18 +02:00
Dominique Eifländer	00bf9f279e	Merge branch 'hotfix-reading-order' into 'main' hotfix: use center coordinates See merge request fforesight/layout-parser!191	2024-08-09 15:51:42 +02:00
Kilian Schuettler	d16377a24a	hotfix: line comparison with center coordinates	2024-08-09 15:45:23 +02:00
Dominique Eifländer	81179ee744	Merge branch 'RED-9760-dcold' into 'main' RED-9760: Changed lineSeparation threshold for documine old See merge request fforesight/layout-parser!190	2024-08-09 14:50:39 +02:00
Dominique Eifländer	1953b5924f	RED-9760: Changed lineSeparation threshold for documine old	2024-08-09 14:42:14 +02:00
Kilian Schüttler	6f6e8d5d4e	Merge branch 'hotfix-reading-order' into 'main' hotfix reading order See merge request fforesight/layout-parser!187	2024-08-09 11:49:12 +02:00
Kilian Schüttler	69bcd4f68d	hotfix reading order	2024-08-09 11:49:12 +02:00
Timo Bejan	b900cfaf31	Merge branch 'CLARI-140' into 'main' CLARI-140 - case issue See merge request fforesight/layout-parser!189	2024-08-08 21:49:40 +02:00
Timo Bejan	cdc2081785	CLARI-140 - case issue	2024-08-08 22:40:11 +03:00
Timo Bejan	a9287ec406	Merge branch 'CLARI-139' into 'main' CLAR-139 - fixed outline error for unparsable object See merge request fforesight/layout-parser!188	2024-08-08 16:36:41 +02:00
Timo Bejan	5b6a706c28	CLAR-139 - fixed outline error for unparsable object	2024-08-08 16:20:14 +03:00
Timo Bejan	28d8ad0a3f	Merge branch 'CLARI-128' into 'main' Fixed Index out of bounds exception in blockificationpostprocessingservice -... See merge request fforesight/layout-parser!186	2024-07-30 16:56:21 +02:00
Timo Bejan	0c1583c1be	Fixed Index out of bounds exception in blockificationpostprocessingservice - this could should be documented btw, there are also probably other use-cases where the code doesnt work	2024-07-30 17:45:05 +03:00