Merge branch 'RED-10270' into 'release/0.188.x'

RED-10270: fix NumberFormatException See merge request fforesight/layout-parser!249
RED-10270: fix NumberFormatException
2024-10-24 17:14:50 +02:00 · 2024-10-24 10:59:30 +02:00 · 2024-10-18 12:20:15 +02:00 · 2024-10-18 12:20:15 +02:00 · 2024-10-18 12:12:15 +02:00 · 2024-10-18 12:12:15 +02:00
72 changed files with 1805 additions and 588 deletions
@@ -2,11 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor;

 import static java.lang.String.format;

+import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -24,6 +26,9 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.No
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.MarkdownMapper;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
@@ -39,7 +44,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableO
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
@@ -47,7 +52,6 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
@@ -57,9 +61,6 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClarifyndClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.Box;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.GraphicExtractorService;
@@ -91,10 +92,7 @@ public class LayoutParsingPipeline {
    CvTableParsingAdapter cvTableParsingAdapter;
    LayoutParsingStorageService layoutParsingStorageService;
    SectionsBuilderService sectionsBuilderService;
-    RedactManagerClassificationService redactManagerClassificationService;
-    DocuMineClassificationService docuMineClassificationService;
    SimplifiedSectionTextService simplifiedSectionTextService;
-    BodyTextFrameService bodyTextFrameService;
    RulingCleaningService rulingCleaningService;
    TableExtractionService tableExtractionService;
    DocuMineBlockificationService docuMineBlockificationService;
@@ -104,12 +102,12 @@ public class LayoutParsingPipeline {
    LayoutGridService layoutGridService;
    ObservationRegistry observationRegistry;
    VisualLayoutParsingAdapter visualLayoutParsingAdapter;
-    ClarifyndClassificationService clarifyndClassificationService;
    GraphicExtractorService graphicExtractorService;
    OutlineExtractorService outlineExtractorService;
    OutlineValidationService outlineValidationService;
    TOCEnrichmentService tocEnrichmentService;
    LayoutparserSettings settings;
+    ClassificationService classificationService;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -272,7 +270,10 @@ public class LayoutParsingPipeline {
            stripper.setEndPage(pageNumber);
            stripper.setPdpage(pdPage);
            stripper.getText(originDocument);
-            List<TextPositionSequence> words = stripper.getTextPositionSequences();
+            List<Word> words = stripper.getWords();
+
+//            rotateDirAdjExactly(words, pdPage); // works really well for many highly rotated documents (e.g. VV-331340.pdf), but it decreases the headline performance by 1.3%, so I am leaving it out for now
+
            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
                var lines = TextPositionOperations.groupByLine(new HashSet<>(words));
                classificationDocument.getLayoutDebugLayer().addLineVisualizationsFromNestedTextPosition(lines, pageNumber);
@@ -295,7 +296,7 @@ public class LayoutParsingPipeline {

            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(words, cleanRulings);

-            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getTextPositionSequences(), false);
+            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getWords(), false);

            pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>())
                    .addAll(graphics.stream()
@@ -308,7 +309,7 @@ public class LayoutParsingPipeline {

            ClassificationPage classificationPage = switch (layoutParsingType) {
                case REDACT_MANAGER_OLD ->
-                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings, classificationDocument.getLayoutDebugLayer());
+                        redactManagerBlockificationService.blockify(stripper.getWords(), cleanRulings, classificationDocument.getLayoutDebugLayer());
                case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
                case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
                        docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
@@ -366,33 +367,9 @@ public class LayoutParsingPipeline {

        originDocument.close();

-        log.info("Calculating BodyTextFrame for {}", identifier);
-        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
-        for (ClassificationPage page : classificationDocument.getPages()) {
-            classificationDocument.getLayoutDebugLayer().addCleanRulingVisualization(page.getCleanRulings(), page.getPageNumber());
-        }
-        log.info("Classify TextBlocks for {}", identifier);
-        switch (layoutParsingType) {
-            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
-                    redactManagerClassificationService.classifyDocument(classificationDocument);
-            case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
-        }
+        classificationService.classify(classificationDocument, layoutParsingType, identifier);

-        if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
-            for (ClassificationPage page : classificationDocument.getPages()) {
-                docuMineBlockificationService.mergeblocks(page, page.getCleanRulings().withoutTextRulings(), 0, 10);
-            }
-        }
-
-        List<TextPageBlock> headlines = classificationDocument.getPages()
-                .stream()
-                .flatMap(classificationPage -> classificationPage.getTextBlocks()
-                        .stream()
-                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
-                        .map(tb -> (TextPageBlock) tb))
-                .toList();
-        TableOfContents tableOfContents = outlineValidationService.createToC(headlines);
+        TableOfContents tableOfContents = outlineValidationService.createToC(classificationDocument);
        classificationDocument.setTableOfContents(tableOfContents);

        log.info("Building Sections for {}", identifier);
@@ -406,6 +383,32 @@ public class LayoutParsingPipeline {
    }


+    private static void rotateDirAdjExactly(List<Word> words, PDPage pdPage) {
+
+        for (TextDirection dir : TextDirection.values()) {
+
+            double averageRotation = words.stream()
+                    .map(Word::getTextPositions)
+                    .flatMap(Collection::stream)
+                    .filter(pos -> pos.getDir().equals(dir))
+                    .mapToDouble(RedTextPosition::getExactDir).average().orElse(0);
+
+            if (averageRotation == 0) {
+                continue;
+            }
+
+            AffineTransform rotateInstance = AffineTransform.getRotateInstance(averageRotation, pdPage.getMediaBox().getWidth() / 2, pdPage.getMediaBox().getHeight() / 2);
+
+            for (Word word : words) {
+                if (!dir.equals(word.getDir())) {
+                    continue;
+                }
+                word.transform(rotateInstance);
+            }
+        }
+    }
+
+
    private void addNumberOfPagesToTrace(int numberOfPages, long size) {

        if (observationRegistry.getCurrentObservation() != null) {
@@ -447,10 +450,10 @@ public class LayoutParsingPipeline {
        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
            if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getSequences() == null) {
+                if (((TextPageBlock) textBlock).getWords() == null) {
                    continue;
                }
-                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                for (Word word : ((TextPageBlock) textBlock).getWords()) {
                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
                    classificationPage.getFontCounter().add(word.getFont());
                    classificationPage.getFontSizeCounter().add(word.getFontSize());
@@ -18,7 +18,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.Zon
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;

 import lombok.RequiredArgsConstructor;
@@ -35,7 +35,7 @@ public class DocstrumSegmentationService {
    private final ReadingOrderService readingOrderService;


-    public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
+    public List<Zone> segmentPage(List<Word> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {

        EnumMap<TextDirection, Integer> directionCounts = new EnumMap<>(TextDirection.class);

@@ -78,11 +78,11 @@ public class DocstrumSegmentationService {
    }


-    private List<Zone> computeZones(List<TextPositionSequence> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
+    private List<Zone> computeZones(List<Word> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {

        List<RedTextPosition> positions = textPositions.stream()
                .filter(t -> t.getDir() == direction)
-                .map(TextPositionSequence::getTextPositions)
+                .map(Word::getTextPositions)
                .flatMap(List::stream)
                .toList();

@@ -225,33 +225,31 @@ public abstract class BoundingBox {

    public double horizontalDistance(BoundingBox other) {

-        Rectangle2D left;
-        Rectangle2D right;
-        if (this.leftOf(other)) {
-            left = this.getBBox();
-            right = other.getBBox();
-        } else {
-            left = other.getBBox();
-            right = this.getBBox();
-        }
+        double rect1Right = getMaxX();
+        double rect1Left = getMinX();
+        double rect2Right = other.getMaxX();
+        double rect2Left = other.getMinX();

-        return Math.max(0, right.getMinX() - left.getMaxX());
+        if (rect1Left > rect2Right || rect2Left > rect1Right) {
+            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        } else {
+            return 0;
+        }
    }


    public double verticalDistance(BoundingBox other) {

-        Rectangle2D bottom;
-        Rectangle2D top;
-        if (this.isAbove(other)) {
-            top = this.getBBox();
-            bottom = other.getBBox();
-        } else {
-            bottom = this.getBBox();
-            top = other.getBBox();
-        }
+        double rect1Top = getMaxY();
+        double rect1Bottom = getMinY();
+        double rect2Top = other.getMaxY();
+        double rect2Bottom = other.getMinY();

-        return Math.max(0, bottom.getMinY() - top.getMaxY());
+        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
+            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        } else {
+            return 0;
+        }
    }


@@ -1,9 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;

-import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.BOLD;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.BOLD_ITALIC;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.ITALIC;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.STANDARD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD_ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.STANDARD;

 import java.util.ArrayList;
 import java.util.Arrays;
@@ -14,7 +14,7 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;

 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;

 import lombok.Data;
@@ -41,7 +41,7 @@ public class Line extends TextBoundingBox {
    private FontStyle fontStyle;

    private final List<Character> characters;
-    private final List<TextPositionSequence> words = new ArrayList<>();
+    private final List<Word> words = new ArrayList<>();


    public Line(List<Character> characters, double wordSpacing) {
@@ -89,7 +89,7 @@ public class Line extends TextBoundingBox {
        for (FontStyle fontStyle : FontStyle.values()) {
            fontStyleCounter.put(fontStyle, new AtomicInteger(0));
        }
-        for (TextPositionSequence word : words) {
+        for (Word word : words) {
            switch (word.getFontStyle()) {
                case STANDARD -> fontStyleCounter.get(FontStyle.REGULAR).getAndIncrement();
                case BOLD -> fontStyleCounter.get(FontStyle.BOLD).getAndIncrement();
@@ -159,14 +159,14 @@ public class Line extends TextBoundingBox {

    private void computeWords(double wordSpacing) {

-        TextPositionSequence word = new TextPositionSequence();
+        Word word = new Word();
        Character previous = null;
        for (Character current : characters) {
            if (previous != null) {
                double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
                if (dist > wordSpacing) {
                    words.add(word);
-                    word = new TextPositionSequence();
+                    word = new Word();
                }
            }
            word.getTextPositions().add(current.getTextPosition());
@@ -99,4 +99,82 @@ public abstract class TextBoundingBox extends BoundingBox {
        return this.bBoxDirAdj.getCenterX();
    }

+
+    public double horizontalDistanceDirAdj(TextBoundingBox other) {
+
+        double rect1Right = getMaxXDirAdj();
+        double rect1Left = getXDirAdj();
+        double rect2Right = other.getMaxXDirAdj();
+        double rect2Left = other.getXDirAdj();
+
+        if (rect1Left > rect2Right || rect2Left > rect1Right) {
+            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public double verticalDistanceDirAdj(TextBoundingBox other) {
+
+        double rect1Top = getMaxYDirAdj();
+        double rect1Bottom = getYDirAdj();
+        double rect2Top = other.getMaxYDirAdj();
+        double rect2Bottom = other.getYDirAdj();
+
+        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
+            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public boolean intersectsDirAdj(TextBoundingBox other) {
+
+        return this.intersectsXDirAdj(other) && this.intersectsYDirAdj(other);
+    }
+
+
+    public boolean intersectsDirAdj(TextBoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsXDirAdj(other, xThreshold) && this.intersectsYDirAdj(other, yThreshold);
+    }
+
+
+    public boolean intersectsXDirAdj(TextBoundingBox other, float threshold) {
+
+        return this.getXDirAdj() - threshold <= other.getMaxXDirAdj() && this.getMaxXDirAdj() + threshold >= other.getXDirAdj();
+    }
+
+
+    public boolean intersectsXDirAdj(TextBoundingBox other) {
+
+        return this.getXDirAdj() <= other.getMaxXDirAdj() && this.getMaxXDirAdj() >= other.getXDirAdj();
+    }
+
+
+    public boolean intersectsYDirAdj(TextBoundingBox other) {
+
+        return this.getYDirAdj() <= other.getMaxYDirAdj() && this.getMaxYDirAdj() >= other.getYDirAdj();
+    }
+
+
+    public boolean intersectsYDirAdj(TextBoundingBox other, float threshold) {
+
+        return this.getYDirAdj() - threshold <= other.getMaxYDirAdj() && this.getMaxYDirAdj() + threshold >= other.getYDirAdj();
+    }
+
+
+    public boolean isAboveDirAdj(TextBoundingBox other) {
+
+        return other.isBelow(this);
+    }
+
+
+    public boolean isBelowDirAdj(TextBoundingBox other) {
+
+        return this.intersectsXDirAdj(other) && this.getYDirAdj() >= other.getMaxYDirAdj();
+    }
+
 }
@@ -28,4 +28,10 @@ public class UnionFind<T> extends org.jgrapht.alg.util.UnionFind<T> {
        return setRep.values();
    }

+
+    public Collection<T> getElements() {
+
+        return getParentMap().keySet();
+    }
+
 }
@@ -44,14 +44,14 @@ public class FloatFrequencyCounter {

    public Double getMostPopular() {

-        if (changed) {
+        if (changed || mostPopularCache == null) {
            Map.Entry<Double, Integer> mostPopular = null;
            for (Map.Entry<Double, Integer> entry : countPerValue.entrySet()) {
                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
                    mostPopular = entry;
                }
            }
-            mostPopularCache = mostPopular != null ? mostPopular.getKey() : null;
+            mostPopularCache = mostPopular != null ? mostPopular.getKey() : 0;
            changed = false;
        }

@@ -3,7 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,8 +16,8 @@ import lombok.experimental.FieldDefaults;
 public class LineInformation {

    List<Rectangle2D> lineBBox;
-    List<List<TextPositionSequence>> sequencesByLines;
+    List<List<Word>> sequencesByLines;
    List<List<Rectangle2D>> bBoxWithGapsByLines;
-    List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
+    List<List<List<Word>>> sequencesWithGapsByLines;

 }
@@ -9,12 +9,13 @@ public enum PageBlockType {
    H6,
    HEADER,
    FOOTER,
-    TITLE,
    PARAGRAPH,
    PARAGRAPH_BOLD,
    PARAGRAPH_ITALIC,
    PARAGRAPH_UNKNOWN,
    OTHER,
+    TABLE_OF_CONTENTS_ITEM,
+    LIST_ITEM,
    TABLE;


@@ -4,7 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -15,7 +15,7 @@ import lombok.Getter;
@AllArgsConstructor
 public class PageContents {

-    List<TextPositionSequence> sortedTextPositionSequences;
+    List<Word> sortedWords;
    Rectangle2D cropBox;
    Rectangle2D mediaBox;
    List<Ruling> rulings;
@@ -16,8 +16,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {

-    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
-    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d{1,2})[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?");
+    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d{1,2})[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?");

    public enum Format {
        EMPTY,
@@ -98,10 +98,10 @@ public class TextRange implements Comparable<TextRange> {
    public List<TextRange> split(List<Integer> splitIndices) {

        if (splitIndices.stream()
-                .anyMatch(idx -> !this.containsExclusive(idx))) {
+                .anyMatch(idx -> !this.contains(idx))) {
            throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s",
                                                       splitIndices.stream()
-                                                               .filter(idx -> !this.containsExclusive(idx))
+                                                               .filter(idx -> !this.contains(idx))
                                                               .toList(),
                                                       this));
        }
@@ -116,7 +116,9 @@ public class TextRange implements Comparable<TextRange> {
            splitBoundaries.add(new TextRange(previousIndex, splitIndex));
            previousIndex = splitIndex;
        }
-        splitBoundaries.add(new TextRange(previousIndex, end));
+        if (previousIndex != end) {
+            splitBoundaries.add(new TextRange(previousIndex, end));
+        }
        return splitBoundaries;
    }

@@ -10,6 +10,7 @@ import java.util.TreeSet;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;

 import io.micrometer.observation.annotation.Observed;
@@ -20,7 +21,9 @@ import lombok.extern.slf4j.Slf4j;
 public class OutlineValidationService {

    @Observed(name = "OutlineValidationService", contextualName = "create-toc")
-    public TableOfContents createToC(List<TextPageBlock> headlines) {
+    public TableOfContents createToC(ClassificationDocument classificationDocument) {
+
+        List<TextPageBlock> headlines = extractHeadlines(classificationDocument);

        List<TableOfContentItem> mainSections = new ArrayList<>();
        Map<Integer, TableOfContentItem> lastItemsPerDepth = new HashMap<>();
@@ -60,4 +63,16 @@ public class OutlineValidationService {
        return new TableOfContents(mainSections);
    }

+
+    private static List<TextPageBlock> extractHeadlines(ClassificationDocument classificationDocument) {
+
+        return classificationDocument.getPages()
+                .stream()
+                .flatMap(classificationPage -> classificationPage.getTextBlocks()
+                        .stream()
+                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
+                        .map(tb -> (TextPageBlock) tb))
+                .toList();
+    }
+
 }
@@ -9,7 +9,7 @@ import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.Data;
@@ -68,12 +68,12 @@ public class Cell extends BoundingBox {
        StringBuilder sb = new StringBuilder();

        Iterator<TextPageBlock> itty = textBlocks.iterator();
-        TextPositionSequence previous = null;
+        Word previous = null;
        while (itty.hasNext()) {

            TextPageBlock textBlock = itty.next();

-            for (TextPositionSequence word : textBlock.getSequences()) {
+            for (Word word : textBlock.getWords()) {
                if (previous != null) {
                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                        sb.append('\n');
@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+
+public record AbstractBlockOnPage(AbstractPageBlock block, ClassificationPage page) {
+
+}
@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@NoArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FrequencyCounters {
+
+    FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
+    FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
+    FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
+    StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
+    StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
+
+}
@@ -0,0 +1,107 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class ListIdentifier {
+
+    public static final Pattern NUMBER_WITH_DOT = Pattern.compile("^\\s*([1-9]{1,4})\\.\\s+");
+    public static final Pattern NUMBER_IN_PARENTHESES = Pattern.compile("^\\s*\\(([1-9]{1,4})\\)\\s+");
+
+    enum Format {
+        NUMBER_WITH_DOT,
+        NUMBER_IN_PARENTHESES
+    }
+
+    Format format;
+    @Getter
+    Word word;
+    @Getter
+    int page;
+    int representation;
+
+
+    public static Optional<ListIdentifier> parse(TextPageBlock textPageBlock, int page) {
+
+        return parse(textPageBlock.getWords().subList(0, Math.min(5, textPageBlock.getWords().size())), page);
+    }
+
+
+    public static Optional<ListIdentifier> parse(List<Word> sequences, int page) {
+
+        StringBuilder sb = new StringBuilder();
+        for (Word sequence : sequences) {
+            sb.append(sequence.toString());
+            sb.append(" ");
+        }
+        sb.replace(sb.length() - 1, sb.length(), "");
+        String text = sb.toString();
+
+        Matcher numberMatcher = NUMBER_WITH_DOT.matcher(text);
+
+        if (numberMatcher.find()) {
+            Optional<Integer> representation = parseInteger(numberMatcher.group(1));
+            if (representation.isPresent()) {
+                return Optional.of(new ListIdentifier(Format.NUMBER_WITH_DOT, sequences.get(0), page, representation.get()));
+            }
+        }
+
+        Matcher parenthesisMatcher = NUMBER_IN_PARENTHESES.matcher(text);
+        if (parenthesisMatcher.find()) {
+            Optional<Integer> representation = parseInteger(parenthesisMatcher.group(1));
+            if (representation.isPresent()) {
+                return Optional.of(new ListIdentifier(Format.NUMBER_IN_PARENTHESES, sequences.get(0), page, representation.get()));
+            }
+        }
+        return Optional.empty();
+    }
+
+
+    private static Optional<Integer> parseInteger(String text) {
+
+        try {
+            return Optional.of(Integer.parseInt(text));
+        } catch (NumberFormatException e) {
+            return Optional.empty();
+        }
+    }
+
+
+    public static boolean isInOrder(List<ListIdentifier> listIdentifiers) {
+
+        if (listIdentifiers.size() <= 1) {
+            return true;
+        }
+
+        for (int i = 1; i < listIdentifiers.size(); i++) {
+            ListIdentifier current = listIdentifiers.get(i);
+            ListIdentifier previous = listIdentifiers.get(i - 1);
+            if (current.format != previous.format) {
+                return false;
+            }
+            if (current.representation <= previous.representation) {
+                return false;
+            }
+            if (!current.word.intersectsXDirAdj(previous.word, 2)) {
+                return false;
+            }
+            if (current.page == previous.page && !current.word.isBelowDirAdj(previous.word)) {
+                return false;
+            }
+            if (current.page < previous.page) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+}
@@ -7,6 +7,8 @@ import org.apache.pdfbox.text.TextPosition;

 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -10,18 +10,18 @@ import lombok.Getter;
@Getter
 public class SearchableText {

-    private final List<TextPositionSequence> sequences = new ArrayList<>();
+    private final List<Word> sequences = new ArrayList<>();


-    public void add(TextPositionSequence textPositionSequence) {
+    public void add(Word word) {

-        sequences.add(textPositionSequence);
+        sequences.add(word);
    }


-    public void addAll(List<TextPositionSequence> textPositionSequences) {
+    public void addAll(List<Word> words) {

-        sequences.addAll(textPositionSequences);
+        sequences.addAll(words);
    }


@@ -32,10 +32,10 @@ public class SearchableText {
    }


-    public static String buildString(List<TextPositionSequence> sequences) {
+    public static String buildString(List<Word> sequences) {

        StringBuilder sb = new StringBuilder();
-        for (TextPositionSequence word : sequences) {
+        for (Word word : sequences) {
            sb.append(word);
            sb.append(' ');
        }
@@ -9,10 +9,14 @@ public class StringFrequencyCounter {

    @Getter
    private final Map<String, Integer> countPerValue = new HashMap<>();
+    boolean changed;
+    String mostPopularCache;


    public void add(String value) {

+        changed = true;
+
        if (!countPerValue.containsKey(value)) {
            countPerValue.put(value, 1);
        } else {
@@ -23,6 +27,8 @@ public class StringFrequencyCounter {

    public void addAll(Map<String, Integer> otherCounter) {

+        changed = true;
+
        for (Map.Entry<String, Integer> entry : otherCounter.entrySet()) {
            if (countPerValue.containsKey(entry.getKey())) {
                countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
@@ -35,13 +41,18 @@ public class StringFrequencyCounter {

    public String getMostPopular() {

-        Map.Entry<String, Integer> mostPopular = null;
-        for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
-                mostPopular = entry;
+        if (changed || mostPopularCache == null) {
+            Map.Entry<String, Integer> mostPopular = null;
+            for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
+                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
+                    mostPopular = entry;
+                }
            }
+            mostPopularCache = mostPopular != null ? mostPopular.getKey() : null;
+            changed = false;
        }
-        return mostPopular != null ? mostPopular.getKey() : null;
+
+        return mostPopularCache;
    }

 }
@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+
+public record TextBlockOnPage(TextPageBlock textBlock, ClassificationPage page) {
+
+}
@@ -7,7 +7,6 @@ import java.util.List;

 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
@@ -26,24 +25,14 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {

    @Builder.Default
-    private List<TextPositionSequence> sequences = new ArrayList<>();
+    private List<Word> words = new ArrayList<>();
+    @Builder.Default
+    private FrequencyCounters frequencyCounters = new FrequencyCounters();

    private Rectangle2D bBoxDirAdj;

-    private String mostPopularWordFont;
-
-    private String mostPopularWordStyle;
-
-    private double mostPopularWordFontSize;
-
-    private double mostPopularWordHeight;
-
-    private double mostPopularWordSpaceWidth;
-
    private boolean underlined;

-    private double highestFontSize;
-
    private PageBlockType classification;

    private boolean toDuplicate;
@@ -52,41 +41,43 @@ public class TextPageBlock extends AbstractPageBlock {
    private boolean changed;


-    public TextPageBlock(List<TextPositionSequence> sequences) {
+    public TextPageBlock(List<Word> words) {

-        this.sequences = new ArrayList<>(sequences);
-        if (!sequences.isEmpty()) {
-            calculateFrequencyCounters();
+        this.words = new ArrayList<>(words);
+        this.frequencyCounters = new FrequencyCounters();
+
+        if (!words.isEmpty()) {
+            addToFrequencyCounters(words);
        }
        calculateBBox();
    }


-    public List<TextPositionSequence> getSequences() {
+    public List<Word> getWords() {

-        return Collections.unmodifiableList(sequences);
+        return Collections.unmodifiableList(words);
    }


    public TextDirection getDir() {

-        return sequences.get(0).getDir();
+        return words.get(0).getDir();
    }


    private void calculateBBox() {

-        if (sequences == null) {
+        if (words == null) {
            this.bBox = new Rectangle2D.Double();
            this.bBoxPdf = new Rectangle2D.Double();
            this.bBoxDirAdj = new Rectangle2D.Double();
            return;
        }
-        this.bBoxDirAdj = sequences.stream()
-                .map(TextPositionSequence::getBBoxDirAdj)
+        this.bBoxDirAdj = words.stream()
+                .map(Word::getBBoxDirAdj)
                .collect(RectangleTransformations.collectBBox());

-        setToBBoxOfComponents(sequences);
+        setToBBoxOfComponents(words);
    }


@@ -108,8 +99,8 @@ public class TextPageBlock extends AbstractPageBlock {
            throw new IllegalArgumentException("Cannot merge textBlocks on different pages.");
        }

-        List<TextPositionSequence> sequences = textBlocksToMerge.stream()
-                .map(TextPageBlock::getSequences)
+        List<Word> sequences = textBlocksToMerge.stream()
+                .map(TextPageBlock::getWords)
                .flatMap(java.util.Collection::stream)
                .toList();
        sequences = new ArrayList<>(sequences);
@@ -118,41 +109,27 @@ public class TextPageBlock extends AbstractPageBlock {
    }


-    private void calculateFrequencyCounters() {
+    private void addToFrequencyCounters(List<Word> sequences) {

-        FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
-        FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
-        FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
-        StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
-        StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
-
-        for (TextPositionSequence wordBlock : sequences) {
-
-            lineHeightFrequencyCounter.add(wordBlock.getTextHeight());
-            fontSizeFrequencyCounter.add(wordBlock.getFontSize());
-            spaceFrequencyCounter.add(wordBlock.getSpaceWidth());
-            fontFrequencyCounter.add(wordBlock.getFont());
-            styleFrequencyCounter.add(wordBlock.getFontStyle());
+        for (Word wordBlock : sequences) {

+            frequencyCounters.getLineHeightFrequencyCounter().add(wordBlock.getTextHeight());
+            frequencyCounters.getFontSizeFrequencyCounter().add(wordBlock.getFontSize());
+            frequencyCounters.getSpaceFrequencyCounter().add(wordBlock.getSpaceWidth());
+            frequencyCounters.getFontFrequencyCounter().add(wordBlock.getFont());
+            frequencyCounters.getStyleFrequencyCounter().add(wordBlock.getFontStyle());
        }

-        setMostPopularWordFont(fontFrequencyCounter.getMostPopular());
-        setMostPopularWordStyle(styleFrequencyCounter.getMostPopular());
-        setMostPopularWordFontSize(fontSizeFrequencyCounter.getMostPopular());
-        setMostPopularWordHeight(lineHeightFrequencyCounter.getMostPopular());
-        setMostPopularWordSpaceWidth(spaceFrequencyCounter.getMostPopular());
-        setHighestFontSize(fontSizeFrequencyCounter.getHighest());
-
-        setUnderlined(sequences.stream()
-                              .allMatch(TextPositionSequence::isUnderline));
+        setUnderlined(this.words.stream()
+                              .allMatch(Word::isUnderline));
    }


-    public TextPageBlock union(TextPositionSequence r) {
+    public TextPageBlock union(Word r) {

        TextPageBlock union = this.copy();
        union.add(r);
-        calculateFrequencyCounters();
+        addToFrequencyCounters(List.of(r));
        calculateBBox();
        return union;
    }
@@ -161,8 +138,8 @@ public class TextPageBlock extends AbstractPageBlock {
    public TextPageBlock union(TextPageBlock r) {

        TextPageBlock union = this.copy();
-        union.addAll(r.getSequences());
-        calculateFrequencyCounters();
+        union.addAll(r.getWords());
+        addToFrequencyCounters(r.getWords());
        calculateBBox();
        return union;
    }
@@ -171,33 +148,33 @@ public class TextPageBlock extends AbstractPageBlock {
    public void add(TextPageBlock textPageBlock) {

        changed = true;
-        sequences.addAll(textPageBlock.getSequences());
-        calculateFrequencyCounters();
+        words.addAll(textPageBlock.getWords());
+        addToFrequencyCounters(textPageBlock.getWords());
        calculateBBox();
    }


-    public void add(TextPositionSequence textPositionSequence) {
+    public void add(Word word) {

        changed = true;
-        sequences.add(textPositionSequence);
-        calculateFrequencyCounters();
+        words.add(word);
+        addToFrequencyCounters(List.of(word));
        calculateBBox();
    }


-    public void addAll(List<TextPositionSequence> textPositionSequences) {
+    public void addAll(List<Word> words) {

        changed = true;
-        sequences.addAll(textPositionSequences);
-        calculateFrequencyCounters();
+        this.words.addAll(words);
+        addToFrequencyCounters(words);
        calculateBBox();
    }


    public TextPageBlock copy() {

-        return new TextPageBlock(new ArrayList<>(sequences));
+        return new TextPageBlock(new ArrayList<>(words));
    }


@@ -216,8 +193,8 @@ public class TextPageBlock extends AbstractPageBlock {

            StringBuilder sb = new StringBuilder();

-            TextPositionSequence previous = null;
-            for (TextPositionSequence word : sequences) {
+            Word previous = null;
+            for (Word word : words) {
                if (previous != null) {
                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                        sb.append('\n');
@@ -240,8 +217,8 @@ public class TextPageBlock extends AbstractPageBlock {
    public int getNumberOfLines() {

        int numberOfLines = 1;
-        TextPositionSequence previous = null;
-        for (TextPositionSequence word : sequences) {
+        Word previous = null;
+        for (Word word : words) {
            if (previous != null) {
                if (word.getMaxYDirAdj() - previous.getMaxYDirAdj() > word.getTextHeight()) {
                    numberOfLines++;
@@ -253,10 +230,47 @@ public class TextPageBlock extends AbstractPageBlock {
    }


+    public String getMostPopularWordFont() {
+
+        return frequencyCounters.getFontFrequencyCounter().getMostPopular();
+    }
+
+
+    public String getMostPopularWordStyle() {
+
+        return frequencyCounters.getStyleFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordFontSize() {
+
+        return frequencyCounters.getFontSizeFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordHeight() {
+
+        return frequencyCounters.getLineHeightFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getMostPopularWordSpaceWidth() {
+
+        return frequencyCounters.getSpaceFrequencyCounter().getMostPopular();
+    }
+
+
+    public double getHighestFontSize() {
+
+        Double highest = frequencyCounters.getFontSizeFrequencyCounter().getHighest();
+        return highest == null ? 0 : highest;
+    }
+
+
    @Override
    public boolean isEmpty() {

-        return sequences.isEmpty();
+        return words.isEmpty();
    }

 }
@@ -0,0 +1,34 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+import java.util.Comparator;
+import java.util.HashMap;
+
+public class TextPositionSequenceComparator implements Comparator<Word> {
+
+    private HashMap<Word, TextBlockOnPage> lookup;
+
+
+    public TextPositionSequenceComparator(HashMap<Word, TextBlockOnPage> lookup) {
+
+        this.lookup = lookup;
+    }
+
+
+    @Override
+    public int compare(Word number1, Word number2) {
+
+        int page1 = lookup.get(number1).page().getPageNumber();
+        int page2 = lookup.get(number2).page().getPageNumber();
+
+        if (page1 != page2) {
+            return Integer.compare(page1, page2);
+        }
+
+        if (number1.getY() != number2.getY()) {
+            return Double.compare(number1.getY(), number2.getY());
+        }
+
+        return Integer.compare(Integer.parseInt(number1.toString()), Integer.parseInt(number2.toString()));
+    }
+
+}
@@ -2,10 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import static com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition.HEIGHT_PADDING;

+import java.awt.geom.AffineTransform;
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;
@@ -24,12 +27,13 @@ import lombok.extern.slf4j.Slf4j;
@NoArgsConstructor
@AllArgsConstructor
@SuppressWarnings("pmd")
-public class TextPositionSequence extends TextBoundingBox implements CharSequence {
+public class Word extends TextBoundingBox implements CharSequence {

    public static final String STANDARD = "standard";
    public static final String BOLD_ITALIC = "bold, italic";
    public static final String BOLD = "bold";
    public static final String ITALIC = "italic";
+    public static final Pattern FONT_CLEANER = Pattern.compile(",bold|,italic");

    private int page;

@@ -43,7 +47,7 @@ public class TextPositionSequence extends TextBoundingBox implements CharSequenc
    private Integer hashcodeCache;


-    public TextPositionSequence(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
+    public Word(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {

        this.textPositions = textPositions.stream()
                .map(RedTextPosition::fromTextPosition)
@@ -61,7 +65,7 @@ public class TextPositionSequence extends TextBoundingBox implements CharSequenc
    }


-    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
+    public Word(List<RedTextPosition> textPositions, int page) {

        this.textPositions = textPositions;
        this.page = page;
@@ -94,9 +98,9 @@ public class TextPositionSequence extends TextBoundingBox implements CharSequenc


    @Override
-    public TextPositionSequence subSequence(int start, int end) {
+    public Word subSequence(int start, int end) {

-        var textPositionSequence = new TextPositionSequence();
+        var textPositionSequence = new Word();
        textPositionSequence.textPositions = textPositions.subList(start, end);
        textPositionSequence.page = page;
        textPositionSequence.dir = dir;
@@ -122,10 +126,10 @@ public class TextPositionSequence extends TextBoundingBox implements CharSequenc
    }


-    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {
+    public void add(Word word, RedTextPosition textPosition) {

        this.textPositions.add(textPosition);
-        this.page = textPositionSequence.getPage();
+        this.page = word.getPage();
        calculateBBoxAndHashcode();
    }

@@ -154,7 +158,8 @@ public class TextPositionSequence extends TextBoundingBox implements CharSequenc
        if (textPositions.get(0).getFontName() == null) {
            return "none";
        }
-        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
+
+        return FONT_CLEANER.matcher(textPositions.get(0).getFontName().toLowerCase(Locale.ROOT)).replaceAll("");
    }


@@ -194,7 +199,7 @@ public class TextPositionSequence extends TextBoundingBox implements CharSequenc
        if (o == this) {
            return true;
        }
-        if (!(o instanceof TextPositionSequence other)) {
+        if (!(o instanceof Word other)) {
            return false;
        }
        if (!other.canEqual((Object) this)) {
@@ -215,7 +220,7 @@ public class TextPositionSequence extends TextBoundingBox implements CharSequenc
    }


-    protected boolean canEqual(final Object other) {return other instanceof TextPositionSequence;}
+    protected boolean canEqual(final Object other) {return other instanceof Word;}


    public int hashCode() {
@@ -238,5 +243,15 @@ public class TextPositionSequence extends TextBoundingBox implements CharSequenc
        return result;
    }

+
+    public void transform(AffineTransform rotateInstance) {
+
+        for (RedTextPosition textPosition : getTextPositions()) {
+            Rectangle2D exactDirAdjCoordinates = rotateInstance.createTransformedShape(textPosition.getBBoxDirAdj()).getBounds2D();
+            textPosition.setBBoxDirAdj(exactDirAdjCoordinates);
+        }
+        calculateBBoxAndHashcode();
+    }
+
 }

@@ -23,11 +23,11 @@ public class DividingColumnDetectionService {
    public List<Rectangle2D> detectColumns(PageContents pageContents) {


-        if (pageContents.getSortedTextPositionSequences().size() < 2) {
+        if (pageContents.getSortedWords().size() < 2) {
            return List.of(pageContents.getCropBox());
        }

-        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), pageContents.getCropBox());
+        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), pageContents.getCropBox());

        return detectColumnsFromLines(linesWithGapInformation.getXGaps(), pageContents.getCropBox());
    }
@@ -5,7 +5,7 @@ import java.util.LinkedList;
 import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.AllArgsConstructor;
 import lombok.experimental.UtilityClass;
@@ -18,23 +18,23 @@ public class GapDetectionService {
    private static final double NEW_LINE_FACTOR = 0.2;


-    public static GapInformation findGapsInLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
+    public static GapInformation findGapsInLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {

-        if (sortedTextPositionSequences.isEmpty()) {
+        if (sortedWords.isEmpty()) {
            return new GapInformation();
        }

-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);

        XGapsContext xGapContext = XGapsContext.init(mainBodyTextFrame);
        YGapsContext yGapContext = YGapsContext.init(mainBodyTextFrame);

-        var previousTextPosition = sortedTextPositionSequences.get(0);
+        var previousTextPosition = sortedWords.get(0);
        Rectangle2D rectangle = toRectangle2D(previousTextPosition);

        xGapContext.addGapFromLeftEdgeOfMainBody(rectangle);

-        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
+        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {

            double yDifference = Math.abs(currentTextPosition.getMaxYDirAdj() - previousTextPosition.getMaxYDirAdj());
            double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj());
@@ -59,14 +59,14 @@ public class GapDetectionService {
            }
            previousTextPosition = currentTextPosition;
        }
-        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedTextPositionSequences.get(sortedTextPositionSequences.size() - 1)));
+        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedWords.get(sortedWords.size() - 1)));
        xGapContext.gapsPerLine.add(xGapContext.gapsInCurrentLine);

        return new GapInformation(xGapContext.gapsPerLine, yGapContext.gapsPerLine);
    }


-    private static Rectangle2D toRectangle2D(TextPositionSequence textPosition) {
+    private static Rectangle2D toRectangle2D(Word textPosition) {

        return mirrorY(textPosition.getBBox());
    }
@@ -87,18 +87,18 @@ public class GapDetectionService {
    }


-    private static void assertAllTextPositionsHaveSameDir(List<TextPositionSequence> textPositionSequences) {
+    private static void assertAllTextPositionsHaveSameDir(List<Word> words) {

-        assert textPositionSequences.stream()
-                .map(TextPositionSequence::getDir)
-                .allMatch(a -> a.equals(textPositionSequences.get(0).getDir()));
+        assert words.stream()
+                .map(Word::getDir)
+                .allMatch(a -> a.equals(words.get(0).getDir()));
    }


-    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
+    private static double getAvgTextPositionHeight(List<Word> words) {

-        return textPositionSequences.stream()
-                .mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
+        return words.stream()
+                .mapToDouble(Word::getHeight).average().orElseThrow();
    }


@@ -7,17 +7,17 @@ import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@UtilityClass
 public class InvisibleTableDetectionService {

-    public List<List<Rectangle2D>> detectTable(List<TextPositionSequence> textPositionSequences, Rectangle2D tableBBox) {
+    public List<List<Rectangle2D>> detectTable(List<Word> words, Rectangle2D tableBBox) {

-        LineInformation lineInformation = LineDetectionService.calculateLineInformation(textPositionSequences);
-        GapInformation gaps = GapDetectionService.findGapsInLines(textPositionSequences, tableBBox);
+        LineInformation lineInformation = LineDetectionService.calculateLineInformation(words);
+        GapInformation gaps = GapDetectionService.findGapsInLines(words, tableBBox);
        List<Rectangle2D> gapsAcrossLines = GapsAcrossLinesService.detectXGapsAcrossLines(gaps, tableBBox);
        List<Double> columnXCoords = gapsAcrossLines.stream().map(RectangularShape::getCenterX).toList();
        int colCount = gapsAcrossLines.size();
@@ -7,7 +7,7 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -19,37 +19,37 @@ public class LineDetectionService {
    private static final double X_GAP_FACTOR = 1; // multiplied with average text height, determines the minimum distance of gaps in lines


-    public LineInformation calculateLineInformation(List<TextPositionSequence> sortedTextPositionSequences) {
+    public LineInformation calculateLineInformation(List<Word> sortedWords) {

-        if (sortedTextPositionSequences.isEmpty()) {
+        if (sortedWords.isEmpty()) {
            return LineFactory.init().build();
        }

-        return buildLineInformation(sortedTextPositionSequences);
+        return buildLineInformation(sortedWords);
    }


-    public List<List<Rectangle2D>> findLinesWithGaps(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
+    public List<List<Rectangle2D>> findLinesWithGaps(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {

-        return calculateLineInformation(sortedTextPositionSequences).getBBoxWithGapsByLines();
+        return calculateLineInformation(sortedWords).getBBoxWithGapsByLines();
    }


-    public List<List<TextPositionSequence>> orderByLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
+    public List<List<Word>> orderByLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {

-        return calculateLineInformation(sortedTextPositionSequences).getSequencesByLines();
+        return calculateLineInformation(sortedWords).getSequencesByLines();
    }


-    private static LineInformation buildLineInformation(List<TextPositionSequence> sortedTextPositionSequences) {
+    private static LineInformation buildLineInformation(List<Word> sortedWords) {

-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);

        LineFactory lineFactory = LineFactory.init();

-        var previousTextPosition = sortedTextPositionSequences.get(0);
+        var previousTextPosition = sortedWords.get(0);
        lineFactory.addToCurrentLine(previousTextPosition);
-        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
+        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {
            if (isNewLine(currentTextPosition, previousTextPosition, avgTextPositionHeight) || isSplitByOrientation(currentTextPosition, previousTextPosition)) {
                lineFactory.startNewLine();
            } else if (isXGap(currentTextPosition, previousTextPosition, avgTextPositionHeight)) {
@@ -63,25 +63,25 @@ public class LineDetectionService {
    }


-    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
+    private static double getAvgTextPositionHeight(List<Word> words) {

-        return textPositionSequences.stream().mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
+        return words.stream().mapToDouble(Word::getHeight).average().orElseThrow();
    }


-    private static boolean isXGap(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
+    private static boolean isXGap(Word currentTextPosition, Word previousTextPosition, double avgTextPositionHeight) {

        return Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj()) > (avgTextPositionHeight * X_GAP_FACTOR);
    }


-    private static boolean isSplitByOrientation(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition) {
+    private static boolean isSplitByOrientation(Word currentTextPosition, Word previousTextPosition) {

        return !previousTextPosition.getDir().equals(currentTextPosition.getDir());
    }


-    private static boolean isNewLine(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
+    private static boolean isNewLine(Word currentTextPosition, Word previousTextPosition, double avgTextPositionHeight) {

        return Math.abs(previousTextPosition.getYDirAdj() - currentTextPosition.getYDirAdj()) > avgTextPositionHeight;
    }
@@ -96,13 +96,13 @@ public class LineDetectionService {
        List<List<Rectangle2D>> bBoxWithGapsByLines;
        List<Rectangle2D> bBoxWithGapsInCurrentLine;

-        List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
-        List<List<TextPositionSequence>> sequencesWithGapsInCurrentLine;
+        List<List<List<Word>>> sequencesWithGapsByLines;
+        List<List<Word>> sequencesWithGapsInCurrentLine;

-        List<TextPositionSequence> currentSequencesWithoutGaps;
+        List<Word> currentSequencesWithoutGaps;

-        List<List<TextPositionSequence>> sequencesByLines;
-        List<TextPositionSequence> sequencesInCurrentLine;
+        List<List<Word>> sequencesByLines;
+        List<Word> sequencesInCurrentLine;

        List<List<Rectangle2D>> xGaps;
        List<List<Rectangle2D>> yGaps;
@@ -116,14 +116,14 @@ public class LineDetectionService {
            List<Rectangle2D> bBoxWithGapsInCurrentLine = new LinkedList<>();
            bBoxWithGapsByLines.add(bBoxWithGapsInCurrentLine);

-            List<List<List<TextPositionSequence>>> sequencesWithGapsByLines = new LinkedList<>();
-            List<List<TextPositionSequence>> sequencesWithGapsInCurrentLine = new LinkedList<>();
+            List<List<List<Word>>> sequencesWithGapsByLines = new LinkedList<>();
+            List<List<Word>> sequencesWithGapsInCurrentLine = new LinkedList<>();
            sequencesWithGapsByLines.add(sequencesWithGapsInCurrentLine);
-            List<TextPositionSequence> currentSequencesWithoutGaps = new LinkedList<>();
+            List<Word> currentSequencesWithoutGaps = new LinkedList<>();
            sequencesWithGapsInCurrentLine.add(currentSequencesWithoutGaps);

-            List<List<TextPositionSequence>> sequencesByLines = new LinkedList<>();
-            List<TextPositionSequence> sequencesInCurrentLine = new LinkedList<>();
+            List<List<Word>> sequencesByLines = new LinkedList<>();
+            List<Word> sequencesInCurrentLine = new LinkedList<>();
            sequencesByLines.add(sequencesInCurrentLine);

            return new LineFactory(lineBBox,
@@ -178,13 +178,13 @@ public class LineDetectionService {
        }


-        private Rectangle2D textPositionBBox(List<TextPositionSequence> textPositionSequences) {
+        private Rectangle2D textPositionBBox(List<Word> words) {

-            return RectangleTransformations.rectangle2DBBox(textPositionSequences.stream().map(TextPositionSequence::getBBox).toList());
+            return RectangleTransformations.rectangle2DBBox(words.stream().map(Word::getBBox).toList());
        }


-        public void addToCurrentLine(TextPositionSequence current) {
+        public void addToCurrentLine(Word current) {

            sequencesInCurrentLine.add(current);
            currentSequencesWithoutGaps.add(current);
@@ -13,7 +13,7 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.springframework.core.io.ClassPathResource;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

@@ -40,7 +40,7 @@ public class PageContentExtractor {
                stripper.setPdpage(pdPage);
                stripper.getText(pdDocument);

-                Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
+                Map<Float, List<Word>> sortedTextPositionSequencesPerDir = stripper.getWords()
                        .stream()
                        .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));

@@ -57,7 +57,7 @@ public class PageContentExtractor {
    }


-    public List<TextPositionSequence> sortByDirAccordingToPageRotation(Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir, int rotation) {
+    public List<Word> sortByDirAccordingToPageRotation(Map<Float, List<Word>> sortedTextPositionSequencesPerDir, int rotation) {

        LinkedList<Float> sortedKeys = new LinkedList<>(sortedTextPositionSequencesPerDir.keySet().stream().sorted().toList());

@@ -14,9 +14,9 @@ public class PageInformationService {

    public PageInformation build(PageContents pageContents) {

-        LineInformation lineInformation = LineDetectionService.calculateLineInformation(pageContents.getSortedTextPositionSequences());
+        LineInformation lineInformation = LineDetectionService.calculateLineInformation(pageContents.getSortedWords());
        Rectangle2D mainBodyTextFrame = MainBodyTextFrameExtractionService.calculateMainBodyTextFrame(lineInformation);
-        GapInformation gapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), mainBodyTextFrame);
+        GapInformation gapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), mainBodyTextFrame);

        return new PageInformation(pageContents, lineInformation, mainBodyTextFrame, gapInformation);
    }
@@ -5,7 +5,7 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@@ -17,9 +17,9 @@ public class TextRulingsClassifier {
    private final static double TEXT_BBOX_THRESHOLD_FACTOR = 0.15; // multiplied with text width then subtracted from word width. If ruling covers this width, it is considered as strikethrough/underline.


-    public static void classifyUnderlinedAndStrikethroughText(List<TextPositionSequence> words, CleanRulings cleanRulings) {
+    public static void classifyUnderlinedAndStrikethroughText(List<Word> words, CleanRulings cleanRulings) {

-        for (TextPositionSequence word : words) {
+        for (Word word : words) {
            if (word.getDir().equals(TextDirection.ZERO) || word.getDir().equals(TextDirection.HALF_CIRCLE)) {
                handleHorizontalText(cleanRulings, word);
            } else {
@@ -29,7 +29,7 @@ public class TextRulingsClassifier {
    }


-    private static void handleVerticalText(CleanRulings cleanRulings, TextPositionSequence word) {
+    private static void handleVerticalText(CleanRulings cleanRulings, Word word) {

        float lowerY = (float) (word.getBBoxPdf().getMinY() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
        float upperY = (float) (word.getBBoxPdf().getMaxY() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
@@ -63,7 +63,7 @@ public class TextRulingsClassifier {
    }


-    private static void handleHorizontalText(CleanRulings cleanRulings, TextPositionSequence word) {
+    private static void handleHorizontalText(CleanRulings cleanRulings, Word word) {

        float leftX = (float) (word.getBBoxPdf().getMinX() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
        float rightX = (float) (word.getBBoxPdf().getMaxX() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
@@ -7,7 +7,6 @@ import java.util.List;
 import java.util.ListIterator;
 import java.util.Locale;

-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.springframework.stereotype.Service;

@@ -17,7 +16,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.Data;

@@ -222,14 +222,14 @@ public class BlockificationPostprocessingService {
            headline = sectionIdentifier + headline;
        }

-        WordSequenceResult wordSequenceResult = findWordSequence(blockToSplit.getSequences(), headline);
+        WordSequenceResult wordSequenceResult = findWordSequence(blockToSplit.getWords(), headline);
        if (wordSequenceResult.inSequence.isEmpty() && !headline.equals(title)) {
-            wordSequenceResult = findWordSequence(blockToSplit.getSequences(), title);
+            wordSequenceResult = findWordSequence(blockToSplit.getWords(), title);
        }

        boolean modifiedBlockToSplit = false;
        if (!wordSequenceResult.inSequence.isEmpty()) {
-            blockToSplit.setSequences(wordSequenceResult.inSequence);
+            blockToSplit.setWords(wordSequenceResult.inSequence);
            blockToSplit.recalculateBBox();
            modifiedBlockToSplit = true;
        }
@@ -250,19 +250,19 @@ public class BlockificationPostprocessingService {
    }


-    private static WordSequenceResult findWordSequence(List<TextPositionSequence> textPositionSequences, String text) {
+    private static WordSequenceResult findWordSequence(List<Word> words, String text) {

        String target = sanitizeString(text);
-        List<TextPositionSequence> inSequence = new ArrayList<>();
-        List<TextPositionSequence> preSequence = new ArrayList<>();
-        List<TextPositionSequence> postSequence = new ArrayList<>();
+        List<Word> inSequence = new ArrayList<>();
+        List<Word> preSequence = new ArrayList<>();
+        List<Word> postSequence = new ArrayList<>();
        StringBuilder currentSequence = new StringBuilder();

        if (target.isBlank()) {
            return new WordSequenceResult();
        }

-        for (TextPositionSequence sequence : textPositionSequences) {
+        for (Word sequence : words) {

            currentSequence.append(sanitizeString(sequence.toString()));
            inSequence.add(sequence);
@@ -274,10 +274,10 @@ public class BlockificationPostprocessingService {
                    int index = 0;
                    String toRemove = currentSequence.substring(0, currentSequence.length() - target.length());

-                    TextPositionSequence next = inSequence.get(index);
+                    Word next = inSequence.get(index);
                    while (currentSequence.length() - next.length() >= target.length()) {

-                        TextPositionSequence removed = inSequence.remove(index);
+                        Word removed = inSequence.remove(index);
                        currentSequence.delete(0, removed.toString().length());
                        preSequence.add(removed);

@@ -306,7 +306,7 @@ public class BlockificationPostprocessingService {
                }

                if (currentSequence.toString().equals(target)) {
-                    postSequence.addAll(textPositionSequences.subList(textPositionSequences.indexOf(sequence) + 1, textPositionSequences.size()));
+                    postSequence.addAll(words.subList(words.indexOf(sequence) + 1, words.size()));
                    return new WordSequenceResult(inSequence, preSequence, postSequence);
                }
            }
@@ -316,10 +316,10 @@ public class BlockificationPostprocessingService {
    }


-    private static SplitSequenceResult splitSequence(TextPositionSequence sequence, String toRemove) {
+    private static SplitSequenceResult splitSequence(Word sequence, String toRemove) {

-        TextPositionSequence in = null;
-        TextPositionSequence out;
+        Word in = null;
+        Word out;

        String currentSequence = sequence.toString().toLowerCase(Locale.ROOT);
        int index = currentSequence.indexOf(toRemove);
@@ -337,9 +337,9 @@ public class BlockificationPostprocessingService {
    }


-    private static TextPositionSequence createSubSequence(TextPositionSequence sequence, int start, int end) {
+    private static Word createSubSequence(Word sequence, int start, int end) {

-        TextPositionSequence newSeq = new TextPositionSequence(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
+        Word newSeq = new Word(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
        newSeq.setParagraphStart(sequence.isParagraphStart());
        return newSeq;
    }
@@ -354,10 +354,10 @@ public class BlockificationPostprocessingService {
            List<TextPageBlock> mergedBlocks = new ArrayList<>();
            for (TextPageBlock textPageBlock : blocksToMerge.subList(1, blocksToMerge.size())) {

-                if (firstBlock != null && !firstBlock.getSequences().isEmpty()) {
+                if (firstBlock != null && !firstBlock.getWords().isEmpty()) {

                    if (textPageBlock.getDir() == firstBlock.getDir()) {
-                        firstBlock.addAll(textPageBlock.getSequences());
+                        firstBlock.addAll(textPageBlock.getWords());
                        mergedBlocks.add(textPageBlock);
                    }
                }
@@ -467,7 +467,9 @@ public class BlockificationPostprocessingService {

    private static String sanitizeString(String text) {

-        return StringUtils.deleteWhitespace(text).toLowerCase(Locale.ROOT);
+        return TextNormalizationUtilities.removeAllWhitespaces(text)//
+                .trim() // sometimes there are trailing empty bytes at the end of the string trim() seems to remove them
+                .toLowerCase(Locale.ENGLISH);
    }


@@ -494,12 +496,12 @@ public class BlockificationPostprocessingService {

    public static class WordSequenceResult {

-        public List<TextPositionSequence> inSequence;
-        public List<TextPositionSequence> preSequence;
-        public List<TextPositionSequence> postSequence;
+        public List<Word> inSequence;
+        public List<Word> preSequence;
+        public List<Word> postSequence;


-        public WordSequenceResult(List<TextPositionSequence> inSequence, List<TextPositionSequence> preSequence, List<TextPositionSequence> postSequence) {
+        public WordSequenceResult(List<Word> inSequence, List<Word> preSequence, List<Word> postSequence) {

            this.inSequence = inSequence;
            this.preSequence = preSequence;
@@ -520,7 +522,7 @@ public class BlockificationPostprocessingService {

    }

-    public record SplitSequenceResult(TextPositionSequence in, TextPositionSequence out) {
+    public record SplitSequenceResult(Word in, Word out) {

    }

@@ -14,7 +14,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;

 import lombok.RequiredArgsConstructor;
@@ -30,7 +30,7 @@ public class DocstrumBlockificationService {
    static final float THRESHOLD = 1f;


-    public ClassificationPage blockify(List<TextPositionSequence> textPositions,
+    public ClassificationPage blockify(List<Word> textPositions,
                                       CleanRulings rulings,
                                       boolean xyOrder,
                                       LayoutDebugLayer visualizations,
@@ -72,16 +72,16 @@ public class DocstrumBlockificationService {
        List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
        zones.forEach(zone -> {

-            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+            List<Word> words = new ArrayList<>();
            zone.getLines()
                    .forEach(line -> {
                        line.getWords()
                                .forEach(word -> {
-                                    textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), word.getPage()));
+                                    words.add(new Word(word.getTextPositions(), word.getPage()));
                                });
                    });

-            abstractPageBlocks.add(buildTextBlock(textPositionSequences, 0));
+            abstractPageBlocks.add(buildTextBlock(words, 0));
        });

        return abstractPageBlocks;
@@ -102,7 +102,7 @@ public class DocstrumBlockificationService {
            }
            TextPageBlock current = (TextPageBlock) block;

-            if (previous != null && !previous.getSequences().isEmpty()) {
+            if (previous != null && !previous.getWords().isEmpty()) {

                if (current.getDir() != previous.getDir() || usedRulings.lineBetween(current, previous)) {
                    previous = current;
@@ -182,8 +182,8 @@ public class DocstrumBlockificationService {

    private TextPageBlock combineBlocksAndResetIterator(TextPageBlock previous, TextPageBlock current, ListIterator<AbstractPageBlock> itty, boolean toDuplicate) {

-        previous.addAll(current.getSequences());
-        previous = buildTextBlock(previous.getSequences(), 0);
+        previous.addAll(current.getWords());
+        previous = buildTextBlock(previous.getWords(), 0);
        previous.setToDuplicate(toDuplicate);
        if (current.getClassification() != null && previous.getClassification() == null) {
            previous.setClassification(current.getClassification());
@@ -283,8 +283,8 @@ public class DocstrumBlockificationService {
                if (current.getDir() == inner.getDir() && current.intersects(inner, yThreshold, xThreshold)) {

                    boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
-                    current.addAll(inner.getSequences());
-                    current = buildTextBlock(current.getSequences(), 0);
+                    current.addAll(inner.getWords());
+                    current = buildTextBlock(current.getWords(), 0);

                    current.setToDuplicate(toDuplicate);
                    blocks.set(i, null);
@@ -301,7 +301,7 @@ public class DocstrumBlockificationService {
    }


-    public static TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    public static TextPageBlock buildTextBlock(List<Word> wordBlockList, int indexOnPage) {

        return new TextPageBlock(wordBlockList);
    }
@@ -16,13 +16,14 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

@SuppressWarnings("all")
@Service
 public class DocuMineBlockificationService {

    static final float THRESHOLD = 1f;
+    public static final double FONT_SIZE_CHANGE_RATIO = 0.15;

    Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);

@@ -36,9 +37,9 @@ public class DocuMineBlockificationService {
     * @param cleanRulings  All rulings on a page
     * @return Page object that contains the Textblock and text statistics.
     */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, CleanRulings cleanRulings) {
+    public ClassificationPage blockify(List<Word> textPositions, CleanRulings cleanRulings) {

-        List<TextPositionSequence> chunkWords = new ArrayList<>();
+        List<Word> chunkWords = new ArrayList<>();
        List<AbstractPageBlock> textPageBlocks = new ArrayList<>();

        CleanRulings usedRulings = cleanRulings.withoutTextRulings();
@@ -47,11 +48,11 @@ public class DocuMineBlockificationService {
        double maxX = 0;
        double minY = 1000;
        double maxY = 0;
-        TextPositionSequence prev = null;
+        Word prev = null;

        boolean wasSplitted = false;
        Double splitX1 = null;
-        for (TextPositionSequence word : textPositions) {
+        for (Word word : textPositions) {

            boolean lineSeparation = prev != null && word.getYDirAdj() - prev.getMaxYDirAdj() > Math.min(word.getHeight(), prev.getHeight()) * 1.1;
            boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
@@ -60,11 +61,7 @@ public class DocuMineBlockificationService {
            boolean newLineAfterSplit = prev != null && word.getYDirAdj() != prev.getYDirAdj() && wasSplitted && splitX1 != word.getXDirAdj();
            boolean isSplitByRuling = prev != null && usedRulings.lineBetween(prev, word);
            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 //
-                                                && (word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold")
-                                                    || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold")
-                                                    || Math.abs(prev.getFontSize() - word.getFontSize()) >= 1
-                                                    || Math.abs(word.getTextHeight() - prev.getTextHeight()) > 0.8);
+            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 && isFontChange(word, prev);

            Matcher matcher = pattern.matcher(chunkWords.stream()
                                                      .collect(Collectors.joining(" ")).toString());
@@ -127,6 +124,15 @@ public class DocuMineBlockificationService {
    }


+    private static boolean isFontChange(Word word, Word prev) {
+
+        return word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold")
+               || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold")
+               || Math.abs(prev.getFontSize() - word.getFontSize()) >= FONT_SIZE_CHANGE_RATIO * Math.min(prev.getFontSize(), word.getFontSize())
+               || Math.abs(word.getTextHeight() - prev.getTextHeight()) >= FONT_SIZE_CHANGE_RATIO * Math.min(prev.getTextHeight(), word.getTextHeight());
+    }
+
+
    public void mergeblocks(ClassificationPage page, CleanRulings usedRulings, float xThreshold, float yThreshold) {

        var blocks = page.getTextBlocks();
@@ -169,8 +175,8 @@ public class DocuMineBlockificationService {
                        .equals(inner.getClassification()))) {

                    boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
-                    current.addAll(inner.getSequences());
-                    current = buildTextBlock(current.getSequences(), 0);
+                    current.addAll(inner.getWords());
+                    current = buildTextBlock(current.getWords(), 0);
                    current.setClassification(inner.getClassification());
                    current.setToDuplicate(toDuplicate);
                    blocks.set(i, null);
@@ -193,7 +199,7 @@ public class DocuMineBlockificationService {
    }


-    public static TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    public static TextPageBlock buildTextBlock(List<Word> wordBlockList, int indexOnPage) {

        return new TextPageBlock(wordBlockList);
    }
@@ -11,7 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;

@SuppressWarnings("all")
@@ -30,20 +30,20 @@ public class RedactManagerBlockificationService {
     * @param visualizations
     * @return Page object that contains the Textblock and text statistics.
     */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, CleanRulings cleanRulings, LayoutDebugLayer visualizations) {
+    public ClassificationPage blockify(List<Word> textPositions, CleanRulings cleanRulings, LayoutDebugLayer visualizations) {

        CleanRulings usedRulings = cleanRulings.withoutTextRulings();

        int indexOnPage = 0;
-        List<TextPositionSequence> chunkWords = new ArrayList<>();
+        List<Word> chunkWords = new ArrayList<>();
        List<AbstractPageBlock> chunkBlockList = new ArrayList<>();

        double minX = 1000, maxX = 0, minY = 1000, maxY = 0;
-        TextPositionSequence prev = null;
+        Word prev = null;

        boolean wasSplitted = false;
        Double splitX1 = null;
-        for (TextPositionSequence word : textPositions) {
+        for (Word word : textPositions) {

            boolean lineSeparation = word.getYDirAdj() - maxY > word.getHeight() * 1.25;
            boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
@@ -10,7 +10,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -60,33 +59,29 @@ public class ClarifyndClassificationService {
            headlineClassificationService.setLastHeadlineFromOutline(textBlock);
            return;
        }
+        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
+                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
+                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
+            return;
+        }
        if (document.getFontSizeCounter().getMostPopular() == null) {
            textBlock.setClassification(PageBlockType.PARAGRAPH);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-            || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
-                                                                                                   || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
-
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                   || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
-                                                                                                           || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
-        } else if (page.getPageNumber() == 1 //
-                   && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
-                       && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
+        if (page.getPageNumber() == 1 //
+            && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
+                && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification(PageBlockType.TITLE);
+                PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+                headlineClassificationService.classifyHeadline(textBlock, headlineType);
+                document.setHeadlines(true);
            }
        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
                   && PositionUtils.getApproxLineCount(textBlock) < 4.9
                   && (textBlock.getMostPopularWordStyle().equals("bold")
                       || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                          && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {

            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
            headlineClassificationService.classifyHeadline(textBlock, headlineType);
@@ -96,7 +91,7 @@ public class ClarifyndClassificationService {
                   && textBlock.getMostPopularWordStyle().equals("bold")
                   && !document.getFontStyleCounter().getMostPopular().equals("bold")
                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {

            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
            headlineClassificationService.classifyHeadline(textBlock, headlineType);
@@ -0,0 +1,29 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import java.util.regex.Pattern;
+
+public class ClassificationPatterns {
+
+    public static final Pattern HEADLINE_WITH_2_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s(?:14C)?\\s*[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN = Pattern.compile("^([0-9]\\.)\\s[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern AT_LEAST_3_CHARS_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern HEADLINE_PATTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
+
+    public static final Pattern AMOUNT_PATTERN = Pattern.compile(
+            "^\\s*\\d+(?:\\.\\d+)?\\s*(?:ml|ul|μl|l|ug|μg|g|kg|mg|cm|cm2|cm3|mm|mm2|mm3|km|km2|m|m2|m3|lb|oz|ppm|dpm|days|weeks|months|%|f|ppb)\\b",
+            Pattern.CASE_INSENSITIVE);
+
+
+
+    public static final Pattern TABLE_OR_FIGURE_PATTERN = Pattern.compile(
+            "^\\s*(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
+            Pattern.CASE_INSENSITIVE);
+
+    public static final Pattern ALPHANUMERIC = Pattern.compile("[a-zA-Z0-9]");
+
+    public static final Pattern NUMERIC = Pattern.compile("[0-9]+");
+
+}
@@ -0,0 +1,62 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class ClassificationService {
+
+    DocuMineBlockificationService docuMineBlockificationService;
+    BodyTextFrameService bodyTextFrameService;
+    TableOfContentsClassificationService tableOfContentsClassificationService;
+    RedactManagerClassificationService redactManagerClassificationService;
+    ClarifyndClassificationService clarifyndClassificationService;
+    DocuMineClassificationService docuMineClassificationService;
+    HeaderFooterClassificationService headerFooterClassificationService;
+
+
+    public void classify(ClassificationDocument document, LayoutParsingType layoutParsingType, Map<String, String> identifier) {
+
+        log.info("Calculating BodyTextFrame for {}", identifier);
+        bodyTextFrameService.setBodyTextFrames(document, layoutParsingType);
+        for (ClassificationPage page : document.getPages()) {
+            document.getLayoutDebugLayer().addCleanRulingVisualization(page.getCleanRulings(), page.getPageNumber());
+        }
+        log.info("Classify TextBlocks for {}", identifier);
+
+        headerFooterClassificationService.classifyHeadersAndFooters(document);
+
+        tableOfContentsClassificationService.classifyTableOfContents(document);
+
+        switch (layoutParsingType) {
+            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
+                    redactManagerClassificationService.classifyDocument(document);
+            case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(document);
+            case CLARIFYND -> clarifyndClassificationService.classifyDocument(document);
+        }
+
+        if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
+            for (ClassificationPage page : document.getPages()) {
+                docuMineBlockificationService.mergeblocks(page, page.getCleanRulings().withoutTextRulings(), 0, 10);
+            }
+        }
+    }
+
+}
@@ -1,10 +1,20 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.classification;

+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.ALPHANUMERIC;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AMOUNT_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AT_LEAST_3_CHARS_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_PATTERN_WITH_SLASHES;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_2_IDENTIFER_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_OR_FIGURE_PATTERN;
+
 import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.regex.Matcher;
-import java.util.regex.Pattern;

 import org.springframework.stereotype.Service;

@@ -12,85 +22,60 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.AbstractBlockOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.HeaderFooterDetection;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DocuMineClassificationService {

-    private static final Pattern HEADLINE_WITH_2_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN = Pattern.compile("^([0-9]\\.)\\s[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern AT_LEAST_3_CHARS_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern HEADLINE_PATTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
-    private static final Pattern AMOUNT_PATTERN = Pattern.compile("^\\s*\\d+(?:\\.\\d+)?\\s*(?:ml|l|g|kg|mg|cm|mm|km|m|lb|oz|ppm|%|f)\\b", Pattern.CASE_INSENSITIVE);
-    private static final Pattern TABLE_OR_FIGURE_PATTER = Pattern.compile(
-            "^\\s*(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
-            Pattern.CASE_INSENSITIVE);
-
    public static final int SEPARATION_THRESHOLD = 10; // if the min distance between a textblock and all its surrounding blocks, the regexes can be more lenient.
    public static final int SURROUNDING_BLOCKS_RADIUS = 3; // number of surrounding blocks before and after the current textblock to be tested

+    ListItemClassificationService listItemClassificationService;
+

    public void classifyDocument(ClassificationDocument document) {

-        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();
-
-        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
+        List<Double> headlineFontSizes = buildHeadlineFontSizes(document);
+        List<AbstractBlockOnPage> blocks = buildBlocksPerPage(document);
+        log.debug("Headline FontSizes are: {}", headlineFontSizes);

        HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();

-        for (ClassificationPage page : document.getPages()) {
-            document.getLayoutDebugLayer().addTextBlockVisualizations(page.getTextBlocks(), page.getPageNumber());
-            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
+        for (int i = 0; i < blocks.size(); i++) {
+            AbstractBlockOnPage block = blocks.get(i);
+            document.getLayoutDebugLayer().addTextBlockVisualizations(block.page().getTextBlocks(), block.page().getPageNumber());
+            classifyBlock(headlineClassificationService, i, blocks, document, headlineFontSizes);
        }
-    }

-
-    private void classifyPage(HeadlineClassificationService headlineClassificationService,
-                              ClassificationPage page,
-                              ClassificationDocument document,
-                              List<Double> headlineFontSizes) {
-
-        List<AbstractPageBlock> textBlocks = page.getTextBlocks();
-        for (int i = 0; i < textBlocks.size(); i++) {
-            AbstractPageBlock textBlock = textBlocks.get(i);
-            if (textBlock instanceof TextPageBlock) {
-                List<AbstractPageBlock> surroundingBlocks = getSurroundingBlocks(i, textBlocks);
-                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, surroundingBlocks, page, document, headlineFontSizes);
-            }
-        }
-    }
-
-
-    private List<AbstractPageBlock> getSurroundingBlocks(int originalIndex, List<AbstractPageBlock> textBlocks) {
-
-        int start = Math.max(originalIndex - SURROUNDING_BLOCKS_RADIUS, 0);
-        int end = Math.min(originalIndex + SURROUNDING_BLOCKS_RADIUS, textBlocks.size());
-        List<AbstractPageBlock> surroundingBlocks = new ArrayList<>(2 * SURROUNDING_BLOCKS_RADIUS);
-        for (int i = start; i < end; i++) {
-            if (i == originalIndex) {
-                continue;
-            }
-            surroundingBlocks.add(textBlocks.get(i));
-        }
-        return surroundingBlocks;
    }


    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
-                               TextPageBlock textBlock,
-                               List<AbstractPageBlock> surroundingBlocks,
-                               ClassificationPage page,
+                               int currentIndex,
+                               List<AbstractBlockOnPage> allBlocks,
                               ClassificationDocument document,
                               List<Double> headlineFontSizes) {

+        TextPageBlock textBlock;
+        if (allBlocks.get(currentIndex).block() instanceof TextPageBlock block) {
+            textBlock = block;
+        } else {
+            return;
+        }
+        ClassificationPage page = allBlocks.get(currentIndex).page();
+        List<AbstractPageBlock> surroundingBlocks = getSurroundingBlocksOnPage(currentIndex, allBlocks);
+
        log.debug("headlineFontSizes: {}", headlineFontSizes);
        var bodyTextFrame = page.getBodyTextFrame();

@@ -98,72 +83,57 @@ public class DocuMineClassificationService {
        Matcher atLeast3Matcher = AT_LEAST_3_CHARS_PATTERN.matcher(textBlock.toString());
        Matcher headlineWithSlashesMatcher = HEADLINE_PATTERN_WITH_SLASHES.matcher(textBlock.toString());
        Matcher amountMatcher = AMOUNT_PATTERN.matcher(textBlock.toString());
-        Matcher tableOrFigureMatcher = TABLE_OR_FIGURE_PATTER.matcher(textBlock.toString());
+        Matcher tableOrFigureMatcher = TABLE_OR_FIGURE_PATTERN.matcher(textBlock.toString());
        Matcher headlineWithSingleIdentifierMatcher = HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN.matcher(textBlock.toString());
        boolean isAtLeast3Characters = atLeast3Matcher.reset().find();
-        boolean isTocItem = textBlock.getText().contains("..............");
        boolean headlineWithSlashesMatches = headlineWithSlashesMatcher.reset().matches();
        boolean isAmount = amountMatcher.reset().find();
        int charCount = countChars(textBlock);

        boolean enoughChars = charCount > textBlock.getText().length() * 0.5;

+        List<ListIdentifier> listIdentifiers = listItemClassificationService.findConfirmedListIdentifiers(currentIndex, allBlocks);
+        document.getLayoutDebugLayer().addListIdentifiers(listIdentifiers);
+
        if (textBlock.getClassification() != null && textBlock.getClassification().isHeadline()) {
            headlineClassificationService.setLastHeadlineFromOutline(textBlock);
            return;
        }
+        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
+                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
+                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
+            return;
+        }
        if (document.getFontSizeCounter().getMostPopular() == null) {
            textBlock.setClassification(PageBlockType.PARAGRAPH);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) //
-            || (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) //
-                && (document.getFontSizeCounter().getMostPopular() == null //
-                    || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular()))) {
-            textBlock.setClassification(PageBlockType.HEADER);
-
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                   || (PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
-                                                          textBlock,
-                                                          page.getRotation())
-                       && (document.getFontSizeCounter().getMostPopular()
-                           == null
-                           || textBlock.getHighestFontSize()
-                              <= document.getFontSizeCounter()
-                                      .getMostPopular()))
-                   || HeaderFooterDetection.isLikelyFooter(textBlock, document, page)) {
-            textBlock.setClassification(PageBlockType.FOOTER);
-        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
-                                                 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
-            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification(PageBlockType.TITLE);
-            }
-        } else if (textBlock.getText().length() > 5
-                   && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular()
-                       || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular())
-                   && PositionUtils.getApproxLineCount(textBlock) < 5.9
-                   && ((textBlock.getMostPopularWordStyle().contains("bold") || textBlock.isUnderlined())//
-                       && Character.isDigit(textBlock.toString().charAt(0)) //
-                       && isAtLeast3Characters //
-                       && !textBlock.toString().contains(":") //
-                       || textBlock.toString().startsWith("APPENDIX") //
-                       || textBlock.toString().startsWith("FIGURE") //
-                       || textBlock.toString().startsWith("Continued TABLE") //
-                       || textBlock.toString().startsWith("TABLE"))
-                   && !textBlock.toString().endsWith(":")
-                   && isAtLeast3Characters
-                   && !isTocItem
-                   && !isAmount
-                   && enoughChars) {
+        if (textBlock.getText().length() > 5
+            && greaterOrEqualFontThanDocumentAverage(textBlock, document)
+            && PositionUtils.getApproxLineCount(textBlock) < 5.9
+            && ((textBlock.getMostPopularWordStyle().contains("bold") || textBlock.isUnderlined())//
+                && Character.isDigit(textBlock.toString().charAt(0)) //
+                && isAtLeast3Characters //
+                && !textBlock.toString().contains(":") //
+                || textBlock.toString().startsWith("APPENDIX") //
+                || textBlock.toString().startsWith("FIGURE") //
+                || textBlock.toString().startsWith("Continued TABLE") //
+                || textBlock.toString().startsWith("TABLE"))
+            && !textBlock.toString().endsWith(":")
+            && isAtLeast3Characters
+            && !isAmount
+            && enoughChars) {

            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
        } else if (isAllCaps(textBlock)
+                   && ALPHANUMERIC.matcher(Character.toString(textBlock.getText().charAt(0))).matches()
+                   && hasSeparation(textBlock, surroundingBlocks)
                   && textBlock.getText().length() > 5
                   && isAtLeast3Characters
                   && !isAmount
                   && enoughChars
                   && !textBlock.toString().contains(":")
-                   && !textBlock.toString().startsWith("(")
+                   && !textBlock.toString().endsWith(".")
                   && PositionUtils.getApproxLineCount(textBlock) < 2.9) {

            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
@@ -171,19 +141,30 @@ public class DocuMineClassificationService {
                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
                   && isAtLeast3Characters
                   && !headlineWithSlashesMatches
-                   && !isAmount
-                   && !isTocItem) {
+                   && !isAmount) {

            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
-        } else if (!isTocItem
-                   && hasSeparation(textBlock, surroundingBlocks)
-                   && greaterOrEqualThanFontPageAverage(textBlock, page)
-                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && (tableOrFigureMatcher.reset().find() || headlineWithSingleIdentifierMatcher.reset().find())
-                   && !isAmount
+        } else if (hasSeparation(textBlock, surroundingBlocks)//
+                   && greaterOrEqualFontThanPageAverage(textBlock, page)//
+                   && PositionUtils.getApproxLineCount(textBlock) < 2.9//
+                   && (tableOrFigureMatcher.reset().find() || (headlineWithSingleIdentifierMatcher.reset().find() && listIdentifiers.isEmpty())) //
+                   && !isAmount//
                   && !headlineWithSlashesMatches) {

            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
+//        } else if (textBlock.getMostPopularWordFont().contains("bold")
+//                   && greaterOrEqualFontThanPageAverage(textBlock, page)
+//                   && textBlock.getWords().size() <= 6
+//                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
+//                   && isAtLeast3Characters
+//                   && charCount > textBlock.getText().length() * 0.75
+//                   && !textBlock.getText().contains(":")
+//                   && textBlock.getWidth() < page.getBodyTextFrame().getWidth() * 0.7) {
+//
+//            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
+        } else if (!listIdentifiers.isEmpty()) {
+
+            textBlock.setClassification(PageBlockType.LIST_ITEM);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                   && textBlock.getMostPopularWordStyle().equals("bold")
@@ -222,13 +203,20 @@ public class DocuMineClassificationService {
    }


-    private static boolean greaterOrEqualThanFontPageAverage(TextPageBlock textBlock, ClassificationPage page) {
+    private static boolean greaterOrEqualFontThanPageAverage(TextPageBlock textBlock, ClassificationPage page) {

        return textBlock.getMostPopularWordHeight() >= page.getTextHeightCounter().getMostPopular() //
               || textBlock.getMostPopularWordFontSize() >= page.getFontSizeCounter().getMostPopular();
    }


+    private static boolean greaterOrEqualFontThanDocumentAverage(TextPageBlock textBlock, ClassificationDocument document) {
+
+        return textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() //
+               || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular();
+    }
+
+
    private static boolean isAllCaps(TextPageBlock textBlock) {

        return textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT));
@@ -246,8 +234,7 @@ public class DocuMineClassificationService {

        return surroundingBlocks.stream()
                .mapToDouble(surroundingBlock -> calculateSeparation(textBlock, surroundingBlock))
-                .min()
-                .orElse(Double.MAX_VALUE);
+                .min().orElse(Double.MAX_VALUE);
    }


@@ -267,6 +254,92 @@ public class DocuMineClassificationService {
        document.setHeadlines(true);
    }

+
+    private List<AbstractBlockOnPage> buildBlocksPerPage(ClassificationDocument document) {
+
+        List<AbstractBlockOnPage> blocks = new ArrayList<>();
+        for (ClassificationPage page : document.getPages()) {
+            for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
+                if (abstractPageBlock instanceof TextPageBlock textBlock) {
+                    if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER) //
+                                                                  || textBlock.getClassification().equals(PageBlockType.FOOTER))) {
+                        continue;
+                    }
+                    blocks.add(new AbstractBlockOnPage(textBlock, page));
+                }
+            }
+        }
+        return blocks;
+    }
+
+
+    private static List<Double> buildHeadlineFontSizes(ClassificationDocument document) {
+
+        if (document.getFontSizeCounter().getCountPerValue().size() <= 6) {
+            return document.getFontSizeCounter().getValuesInReverseOrder();
+        }
+
+        List<Map.Entry<Double, Integer>> sortedEntries = new ArrayList<>(document.getFontSizeCounter().getCountPerValue().entrySet());
+        sortedEntries.sort(Map.Entry.comparingByKey());
+
+        int totalCount = sortedEntries.stream()
+                .mapToInt(Map.Entry::getValue).sum();
+
+        int cumulativeCount = 0;
+        Iterator<Map.Entry<Double, Integer>> iterator = sortedEntries.iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<Double, Integer> entry = iterator.next();
+            cumulativeCount += entry.getValue();
+            if (cumulativeCount > totalCount * 0.3) {
+                break; // We've filtered the bottom 30%, so stop.
+            }
+            iterator.remove();
+        }
+
+        if (sortedEntries.size() < 6) {
+            return document.getFontSizeCounter().getValuesInReverseOrder();
+        }
+        int clusterSize = Math.max(1, sortedEntries.size() / 6);
+
+        List<List<Double>> clusters = new ArrayList<>();
+        for (int i = 0; i < 6; i++) {
+            clusters.add(new ArrayList<>());
+        }
+
+        for (int i = 0; i < sortedEntries.size(); i++) {
+            int clusterIndex = Math.min(i / clusterSize, 5);
+            clusters.get(clusterIndex).add(sortedEntries.get(i).getKey());
+        }
+
+        return clusters.stream()
+                .map(cluster -> cluster.stream()
+                        .mapToDouble(d -> d).average()
+                        .orElseThrow())
+                .sorted(Comparator.reverseOrder())
+                .toList();
+    }
+
+
+    private List<AbstractPageBlock> getSurroundingBlocksOnPage(int originalIndex, List<AbstractBlockOnPage> textBlocks) {
+
+        int start = Math.max(originalIndex - SURROUNDING_BLOCKS_RADIUS, 0);
+        int end = Math.min(originalIndex + SURROUNDING_BLOCKS_RADIUS, textBlocks.size());
+        List<AbstractPageBlock> surroundingBlocks = new ArrayList<>(2 * SURROUNDING_BLOCKS_RADIUS);
+        for (int i = start; i < end; i++) {
+            if (i == originalIndex) {
+                continue;
+            }
+            if (textBlocks.get(i).block().getText().length() <= 1) {
+                continue;
+            }
+            if (!textBlocks.get(i).page().equals(textBlocks.get(originalIndex).page())) {
+                continue;
+            }
+            surroundingBlocks.add(textBlocks.get(i).block());
+        }
+        return surroundingBlocks;
+    }
+
 }


@@ -0,0 +1,55 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class HeaderFooterClassificationService {
+
+    public void classifyHeadersAndFooters(ClassificationDocument document) {
+
+        for (ClassificationPage page : document.getPages()) {
+            for (AbstractPageBlock pageBlock : page.getTextBlocks()) {
+                if (pageBlock instanceof TextPageBlock textBlock) {
+                    classifyBlock(document, page, textBlock);
+                }
+            }
+        }
+
+    }
+
+
+    private static void classifyBlock(ClassificationDocument document, ClassificationPage page, TextPageBlock textBlock) {
+
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+            || PositionUtils.isOverBodyTextFrame(page.getBodyTextFrame(), textBlock, page.getRotation()) && smallerFontThanDocAverage(document, textBlock)) {
+
+            textBlock.setClassification(PageBlockType.HEADER);
+
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                   || PositionUtils.isUnderBodyTextFrame(page.getBodyTextFrame(), textBlock, page.getRotation()) && smallerFontThanDocAverage(document, textBlock)) {
+
+            textBlock.setClassification(PageBlockType.FOOTER);
+        }
+    }
+
+
+    private static boolean smallerFontThanDocAverage(ClassificationDocument document, TextPageBlock textBlock) {
+
+        return document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular();
+    }
+
+}
@@ -73,13 +73,18 @@ public class HeadlineClassificationService {

    public static PageBlockType headlineClassByFontSize(TextPageBlock textBlock, List<Double> fontSizeGroups) {

-        PageBlockType headlineType = PageBlockType.H1;
-        for (int i = 1; i <= fontSizeGroups.size(); i++) {
-            if (textBlock.getMostPopularWordFontSize() == fontSizeGroups.get(i - 1)) {
-                headlineType = PageBlockType.getHeadlineType(i);
+        List<Double> distances = fontSizeGroups.stream()
+                .map(fontSize -> Math.abs(fontSize - textBlock.getMostPopularWordFontSize()))
+                .toList();
+        double min = Double.MAX_VALUE;
+        int argMin = -1;
+        for (int i = 0; i < distances.size(); i++) {
+            if (distances.get(i) < min) {
+                min = distances.get(i);
+                argMin = i;
            }
        }
-        return headlineType;
+        return PageBlockType.getHeadlineType(argMin);
    }

 }
@@ -0,0 +1,99 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.AbstractBlockOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+
+@Service
+public class ListItemClassificationService {
+
+    public static final int LIST_IDENTIFIER_SEARCH_RADIUS = 3;
+
+
+    public List<ListIdentifier> findConfirmedListIdentifiers(int currentIndex, List<AbstractBlockOnPage> allBlocks) {
+
+        List<ListIdentifier> listIdentifiers = extractListIdentifiers(allBlocks.get(currentIndex));
+        if (listIdentifiers.isEmpty()) {
+            return Collections.emptyList();
+        }
+        if (listIdentifiers.size() > 1 && ListIdentifier.isInOrder(listIdentifiers)) {
+            return listIdentifiers;
+        }
+
+        int start = Math.max(0, currentIndex - LIST_IDENTIFIER_SEARCH_RADIUS);
+        int end = Math.min(allBlocks.size(), currentIndex + LIST_IDENTIFIER_SEARCH_RADIUS);
+
+        List<ListIdentifier> identifiersBehind = new ArrayList<>();
+        if (start < currentIndex) {
+            identifiersBehind.addAll(allBlocks.subList(start, currentIndex)
+                                             .stream()
+                                             .map(this::extractListIdentifiers)
+                                             .flatMap(Collection::stream)
+                                             .toList());
+        }
+        if (!identifiersBehind.isEmpty()) {
+            listIdentifiers.add(0, identifiersBehind.get(identifiersBehind.size() - 1));
+            if (ListIdentifier.isInOrder(listIdentifiers)) {
+                return listIdentifiers;
+            }
+            listIdentifiers.remove(0);
+        }
+        List<ListIdentifier> identifiersAhead = new ArrayList<>();
+        if (currentIndex + 1 < end) {
+            identifiersAhead.addAll(allBlocks.subList(currentIndex + 1, end)
+                                            .stream()
+                                            .map(this::extractListIdentifiers)
+                                            .flatMap(Collection::stream)
+                                            .toList());
+        }
+        if (!identifiersAhead.isEmpty()) {
+            listIdentifiers.add(identifiersAhead.get(0));
+            if (ListIdentifier.isInOrder(listIdentifiers)) {
+                return listIdentifiers;
+            }
+            listIdentifiers.remove(listIdentifiers.size() - 1);
+        }
+        return Collections.emptyList();
+
+    }
+
+
+    private List<ListIdentifier> extractListIdentifiers(AbstractBlockOnPage block) {
+
+        List<ListIdentifier> result = new LinkedList<>();
+        if (block.block() instanceof TextPageBlock textBlock) {
+            List<Word> sequences = textBlock.getWords();
+            for (int i = 0; i < sequences.size(); i++) {
+
+                if (i != 0 && sequences.get(i - 1).getXDirAdj() < sequences.get(i).getXDirAdj()) {
+                    // is not the start of a line, continue
+                    continue;
+                }
+
+                Word sequence = sequences.get(i);
+                List<Word> wordsAtStartOfLine = new ArrayList<>(3);
+                int end = Math.min(sequences.size(), i + 3);
+                for (int j = i; j < end; j++) {
+                    if (sequences.get(j).intersectsYDirAdj(sequence, 2)) {
+                        wordsAtStartOfLine.add(sequences.get(j));
+                    } else {
+                        break;
+                    }
+                }
+
+                ListIdentifier.parse(wordsAtStartOfLine, block.page().getPageNumber()).ifPresent(result::add);
+            }
+        }
+        return result;
+    }
+
+}
@@ -11,7 +11,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -61,6 +60,15 @@ public class RedactManagerClassificationService {
            headlineClassificationService.setLastHeadlineFromOutline(textBlock);
            return;
        }
+        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
+                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
+                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
+            return;
+        }
+        if (document.getFontSizeCounter().getMostPopular() == null) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+            return;
+        }
        if (document.getFontSizeCounter().getMostPopular() == null) {
            textBlock.setClassification(PageBlockType.PARAGRAPH);
            return;
@@ -73,31 +81,20 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.PARAGRAPH);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-            || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
-                                                                                                   || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-
-            textBlock.setClassification(PageBlockType.HEADER);
-
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                   || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
-                                                                                                           || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-
-            textBlock.setClassification(PageBlockType.FOOTER);
-        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
-                                                 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
+        if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
+                                          && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {

            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification(PageBlockType.TITLE);
+                PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+                headlineClassificationService.classifyHeadline(textBlock, headlineType);
+                document.setHeadlines(true);
            }
        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
                   && PositionUtils.getApproxLineCount(textBlock) < 4.9
                   && (textBlock.getMostPopularWordStyle().equals("bold")
                       || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                          && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {

            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
            headlineClassificationService.classifyHeadline(textBlock, headlineType);
@@ -107,7 +104,7 @@ public class RedactManagerClassificationService {
                   && textBlock.getMostPopularWordStyle().equals("bold")
                   && !document.getFontStyleCounter().getMostPopular().equals("bold")
                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {

            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
            headlineClassificationService.classifyHeadline(textBlock, headlineType);
@@ -0,0 +1,370 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AMOUNT_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.NUMERIC;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextBlockOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequenceComparator;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
+import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class TableOfContentsClassificationService {
+
+    private static final int MAX_PAGE_COUNT = 10; // maximum length of a toc to avoid runaway classification
+    private static final int SURROUNDING_BLOCKS_RADIUS = 10; // number of blocks to look ahead
+    private static final int MINIMUM_MATCHES = 2; // minimum cluster size
+    public static final int INTERSECTION_TOLERANCE = 2; // equality threshold for x intersection
+    public static final int DENSITY_THRESHOLD_COUNT = 10; // describes the minimum density, at least this many entries per page height are required
+
+
+    @SuppressWarnings("checkstyle:ModifiedControlVariable")
+    public void classifyTableOfContents(ClassificationDocument document) {
+
+        List<TextBlockOnPage> textBlocks = buildBlocksPerPage(document);
+
+        for (int i = 0; i < textBlocks.size(); i++) {
+            TextBlockOnPage textBlock = textBlocks.get(i);
+
+            if (!isTOCHeadline(textBlock)) {
+                continue;
+            }
+
+            int end = identifyTOCItems(i + 1, textBlocks, document);
+
+            if (end > i + 1) {
+                if (textBlock.textBlock().getClassification() == null) {
+                    textBlock.textBlock().setClassification(PageBlockType.H1);
+                }
+                i = end;
+            }
+        }
+    }
+
+
+    private int identifyTOCItems(int start, List<TextBlockOnPage> textBlocks, ClassificationDocument document) {
+
+        ClassificationPage startPage = textBlocks.get(start).page();
+        List<TextBlockOnPage> initialLookAhead = textBlocks.subList(start, Math.min(start + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()));
+        HashMap<Word, TextBlockOnPage> lookup = new HashMap<>();
+        List<Word> numbers = extractNumbers(initialLookAhead, lookup, document.getPages().size());
+        TocNumberFinder tocNumberFinder = new TocNumberFinder(numbers, lookup);
+
+        int lastCandidate = start;
+        for (int i = start; i < Math.min(lastCandidate + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()); i++) {
+
+            TextBlockOnPage textBlockOnPage = textBlocks.get(i);
+            if (textBlockOnPage.page().getPageNumber() - MAX_PAGE_COUNT > startPage.getPageNumber()) {
+                break;
+            }
+
+            if (textBlockOnPage.textBlock().getClassification() != null //
+                && textBlockOnPage.textBlock().getClassification().isHeadline() //
+                && !(textBlockOnPage.textBlock().getText().startsWith("TABLES") //
+                     || textBlockOnPage.textBlock().getText().startsWith("APPENDICES") //
+                     || textBlockOnPage.textBlock().getText().startsWith("FIGURES"))) {
+                log.debug("hit an outline headline, stop immediately.");
+                lastCandidate = i - 1;
+                break;
+            }
+
+            List<Word> numbersFromBlock = extractNumbers(textBlockOnPage, lookup, document.getPages().size());
+
+            List<Word> currentRightmostCluster = tocNumberFinder.getCurrentRightmostCluster();
+
+            if (currentRightmostCluster.size() < MINIMUM_MATCHES) {
+                log.debug("No numbers indicating a table of contents here.");
+                return start;
+            }
+
+            if (anyIntersection(currentRightmostCluster, numbersFromBlock, lookup)) {
+                lastCandidate = i;
+                numbersFromBlock.forEach(tocNumberFinder::add);
+            }
+        }
+
+        addVisualization(document.getLayoutDebugLayer(), tocNumberFinder, lookup);
+
+        Set<TextBlockOnPage> blocksWithNumberInCluster = tocNumberFinder.getCurrentRightmostCluster()
+                .stream()
+                .map(lookup::get)
+                .collect(Collectors.toSet());
+
+        int lastConfirmed = start;
+        for (int i = start; i < lastCandidate + 1; i++) {
+            TextBlockOnPage textBlockOnPage = textBlocks.get(i);
+            if (blocksWithNumberInCluster.contains(textBlockOnPage)) {
+                lastConfirmed = i;
+            }
+        }
+
+        textBlocks.subList(start, lastConfirmed + 1)
+                .stream()
+                .filter(block -> (block.textBlock().getClassification() == null || !block.textBlock().getClassification().isHeadline()))
+                .forEach(textBlockOnPage -> textBlockOnPage.textBlock().setClassification(PageBlockType.TABLE_OF_CONTENTS_ITEM));
+
+        return lastCandidate;
+    }
+
+
+    private static void addVisualization(LayoutDebugLayer layoutDebugLayer, TocNumberFinder tocNumberFinder, Map<Word, TextBlockOnPage> lookup) {
+
+        tocNumberFinder.getCurrentRightmostCluster()
+                .stream()
+                .collect(Collectors.groupingBy(key -> lookup.get(key).page().getPageNumber()))
+                .forEach((pageNumber, number) -> layoutDebugLayer.addTocPages(number, pageNumber));
+    }
+
+
+    private static boolean anyIntersection(Collection<Word> numbers1,
+                                           Collection<Word> numbers2,
+                                           Map<Word, TextBlockOnPage> lookup) {
+
+        return numbers1.stream()
+                .anyMatch(numberFromCluster -> numbers2.stream()
+                        .anyMatch(numberFromBlock -> matches(numberFromBlock, numberFromCluster, lookup)));
+    }
+
+
+    private static List<Word> extractNumbers(List<TextBlockOnPage> textBlocks, Map<Word, TextBlockOnPage> lookup, int numberOfPages) {
+
+        List<Word> blocks = new LinkedList<>();
+        for (TextBlockOnPage textBlock : textBlocks) {
+            blocks.addAll(extractNumbers(textBlock, lookup, numberOfPages));
+        }
+        return blocks;
+    }
+
+
+    private static List<Word> extractNumbers(TextBlockOnPage textBlock, Map<Word, TextBlockOnPage> lookup, int numberOfPages) {
+
+        List<Word> blocks = new LinkedList<>();
+        TextPageBlock block = textBlock.textBlock();
+        List<Word> sequences = block.getWords();
+        for (int i = 0; i < sequences.size(); i++) {
+
+            Word word = sequences.get(i);
+
+            if (!NUMERIC.matcher(word).matches() || word.length() > 5) {
+                continue;
+            }
+
+            if (AMOUNT_PATTERN.matcher(getSurroundingString(i, sequences)).matches()) {
+                continue;
+            }
+
+            try {
+                int pageNumber = Integer.parseInt(word.toString());
+                if (0 >= pageNumber || pageNumber > numberOfPages) {
+                    continue;
+                }
+                lookup.put(word, textBlock);
+                blocks.add(word);
+            } catch (NumberFormatException e) {
+                log.debug("That wasn't a number! Should not happen, due to numeric check beforehand.");
+            }
+        }
+        return blocks;
+    }
+
+
+    private static CharSequence getSurroundingString(int i, List<Word> sequences) {
+
+        int end = Math.min(i + 5, sequences.size());
+        return sequences.subList(i, end)
+                .stream()
+                .map(Word::toString)
+                .collect(Collectors.joining(" "));
+    }
+
+
+    private static boolean matches(Word number1, Word number2, Map<Word, TextBlockOnPage> lookup) {
+
+        if (number1.getDir() != number2.getDir()) {
+            return false;
+        }
+
+        return number1.intersectsXDirAdj(number2, INTERSECTION_TOLERANCE);
+    }
+
+
+    private boolean isTOCHeadline(TextBlockOnPage textBlock) {
+
+        if (textBlock.textBlock().getText().length() > 50) {
+            return false;
+        }
+        String text = TextNormalizationUtilities.removeAllWhitespaces(textBlock.textBlock().getText().toLowerCase(Locale.ENGLISH));
+        return (text.contains("content") && text.length() < "content".length() + 6) //
+               || (text.contains("tableofcontent") && text.length() < "tableofcontent".length() + DENSITY_THRESHOLD_COUNT)//
+               || text.equals("tables")//
+               || text.equals("appendices")//
+               || text.equals("figures");
+    }
+
+
+    private List<TextBlockOnPage> buildBlocksPerPage(ClassificationDocument document) {
+
+        List<TextBlockOnPage> blocks = new ArrayList<>();
+        for (ClassificationPage page : document.getPages()) {
+            for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
+                if (abstractPageBlock instanceof TextPageBlock textBlock) {
+                    if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER) //
+                                                                  || textBlock.getClassification().equals(PageBlockType.FOOTER))) {
+                        continue;
+                    }
+                    blocks.add(new TextBlockOnPage(textBlock, page));
+                }
+            }
+        }
+        return blocks;
+    }
+
+
+    private static class TocNumberFinder {
+
+        final UnionFind<Word> numberClusters;
+        final HashMap<Word, TextBlockOnPage> lookup;
+
+
+        TocNumberFinder(List<Word> blocks, HashMap<Word, TextBlockOnPage> lookup) {
+
+            this.numberClusters = new UnionFind<>(new HashSet<>(blocks));
+            for (int i = 0; i < blocks.size(); i++) {
+                for (int j = i + 1; j < blocks.size(); j++) {
+                    if (matches(blocks.get(i), blocks.get(j), lookup)) {
+                        numberClusters.union(blocks.get(i), blocks.get(j));
+                    }
+                }
+            }
+            this.lookup = lookup;
+        }
+
+
+        public void add(Word number) {
+
+            if (numberClusters.getElements().contains(number)) {
+                return;
+            }
+
+            numberClusters.addElement(number);
+            for (Word element : numberClusters.getElements()) {
+                if (matches(number, element, lookup)) {
+                    numberClusters.union(element, number);
+                }
+            }
+        }
+
+
+        public List<Word> getCurrentRightmostCluster() {
+
+            return numberClusters.getGroups()
+                    .stream()
+                    .filter(cluster -> cluster.size() > MINIMUM_MATCHES)
+                    .map(cluster -> cluster.stream()
+                            .sorted(new TextPositionSequenceComparator(lookup))
+                            .toList())
+                    .map(this::removeOutliers)
+//                    .map(this::filterByMinimumDensity)
+                    .filter(cluster -> cluster.size() > MINIMUM_MATCHES)
+                    .max(Comparator.comparingDouble(cluster -> cluster.get(0).getBBox().getMaxX())).orElse(Collections.emptyList());
+        }
+
+// does not seem to be doing much, ideally instead of using the height of the blocks i would like to use the height, beginning from the MainBody top,
+// but as the MainBody is often wrong, this results in some numbers being discarded even though they are correct.
+//        private List<TextPositionSequence> filterByMinimumDensity(List<TextPositionSequence> numbers) {
+//
+//            Map<ClassificationPage, List<TextPositionSequence>> clustersPerPage = numbers.stream()
+//                    .collect(Collectors.groupingBy(number -> lookup.get(number).page()));
+//
+//            List<TextPositionSequence> result = new ArrayList<>(numbers.size());
+//            clustersPerPage.keySet()
+//                    .stream()
+//                    .sorted(Comparator.comparingInt(ClassificationPage::getPageNumber))
+//                    .forEach(page -> {
+//                        var numbersOnPage = clustersPerPage.get(page);
+//
+//                        double height = numbersOnPage.stream()
+//                                .map(BoundingBox::getBBox)
+//                                .collect(RectangleTransformations.collectBBox()).getHeight();
+//
+//                        double count = numbersOnPage.size();
+//
+//                        if ((count / height) >= (DENSITY_THRESHOLD_COUNT / page.getPageHeight())) {
+//                            result.addAll(numbers);
+//                        }
+//                    });
+//            return result;
+//        }
+
+
+        public List<Word> removeOutliers(List<Word> numbers) {
+
+            List<Word> result = new ArrayList<>();
+
+            result.add(numbers.get(0));
+
+            for (int i = 1; i < numbers.size() - 1; i++) {
+                int prev = getNumberAsInt(numbers, i - 1);
+                int curr = getNumberAsInt(numbers, i);
+                int next = getNumberAsInt(numbers, i + 1);
+
+                if (!(curr <= prev || curr >= next) || !isBetterWithout(numbers, i)) {
+                    result.add(numbers.get(i));
+                }
+            }
+            if (getNumberAsInt(numbers, numbers.size() - 1) >= getNumberAsInt(numbers, Math.max(0, numbers.size() - 2))) {
+                result.add(numbers.get(numbers.size() - 1));
+            }
+
+            return result;
+        }
+
+
+        // Helper method to check if removing the current number results in a better order
+        public static boolean isBetterWithout(List<Word> numbers, int i) {
+
+            if (i == 0 || i == numbers.size() - 1) {
+                return false;
+            }
+
+            int prev = getNumberAsInt(numbers, i - 1);
+            int curr = getNumberAsInt(numbers, i);
+            int next = getNumberAsInt(numbers, i + 1);
+
+            return (prev <= next) && (Math.abs(prev - next) < Math.abs(prev - curr) + Math.abs(curr - next));
+        }
+
+    }
+
+
+    private static int getNumberAsInt(List<Word> numbers, int i) {
+
+        return Integer.parseInt(numbers.get(i).toString());
+    }
+
+}
@@ -97,12 +97,12 @@ public class DocumentGraphFactory {
        for (TableOfContentItem tocItem : classificationDocument.getTableOfContents()) {
            GenericSemanticNode parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
            Optional<GenericSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
-                                                                                   parent,
-                                                                                   tocItem.getChildren().isEmpty(),
-                                                                                   tocItem.getNonEmptySectionBlocks(),
-                                                                                   tocItem.getImages(),
-                                                                                   context,
-                                                                                   document);
+                                                                                  parent,
+                                                                                  tocItem.getChildren().isEmpty(),
+                                                                                  tocItem.getNonEmptySectionBlocks(),
+                                                                                  tocItem.getImages(),
+                                                                                  context,
+                                                                                  document);
            tocItem.setSection(section.orElse(null));
        }
    }
@@ -133,9 +133,9 @@ public class DocumentGraphFactory {

        if (node instanceof DuplicatedParagraph duplicatedParagraph) {
            AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock(textBlocks.stream()
-                                                                                                       .flatMap(tb -> tb.getSequences()
-                                                                                                               .stream())
-                                                                                                       .collect(Collectors.toList()), node, context, page);
+                                                                                                      .flatMap(tb -> tb.getWords()
+                                                                                                              .stream())
+                                                                                                      .collect(Collectors.toList()), node, context, page);
            duplicatedParagraph.setUnsortedLeafTextBlock(unsortedTextBlock);
        }

@@ -12,7 +12,7 @@ import java.util.Objects;

 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@@ -28,7 +28,7 @@ public class SearchTextWithTextPositionFactory {
    public static final double LINEBREAK_DELTA_TOLERANCE = 1.5;


-    public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<TextPositionSequence> sequences) {
+    public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<Word> sequences) {

        if (sequences.isEmpty() || sequences.stream()
                .allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
@@ -40,7 +40,7 @@ public class SearchTextWithTextPositionFactory {
        RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
        RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").bBoxDirAdj(currentTextPosition.getBBoxDirAdj()).build();

-        for (TextPositionSequence word : sequences) {
+        for (Word word : sequences) {
            for (int i = 0; i < word.getTextPositions().size(); ++i) {

                currentTextPosition = word.getTextPositions().get(i);
@@ -66,7 +66,7 @@ public class SearchTextWithTextPositionFactory {
        }

        List<Rectangle2D> positions = sequences.stream()
-                .map(TextPositionSequence::getTextPositions)
+                .map(Word::getTextPositions)
                .flatMap(Collection::stream)
                .map(RedTextPosition::getBBoxPdf)
                .toList();
@@ -16,7 +16,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Ta
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import lombok.experimental.UtilityClass;
@@ -115,7 +115,7 @@ public class TableNodeFactory {
        if (cell.getTextBlocks().isEmpty()) {
            tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page));
        } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page);
+            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getWords(), tableCell, context, page);
            tableCell.setLeafTextBlock(textBlock);
        } else if (firstTextBlockIsHeadline(cell)) {
            SectionNodeFactory.addSection(layoutParsingType,
@@ -129,7 +129,7 @@ public class TableNodeFactory {
                                          context,
                                          document);
        } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
-            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSort(cell.getTextBlocks());
+            List<Word> sequences = TextPositionOperations.mergeAndSort(cell.getTextBlocks());
            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
            tableCell.setLeafTextBlock(textBlock);
        } else {
@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;

 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
@@ -17,14 +17,14 @@ public class TextBlockFactory {
    long textBlockIdx;


-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<Word> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {

        Integer numberOnPage = context.getAndIncrementTextBlockNumberOnPage(page);
        return buildAtomicTextBlock(sequences, parent, numberOnPage, page);
    }


-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, Integer numberOnPage, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<Word> sequences, SemanticNode parent, Integer numberOnPage, Page page) {

        SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionDto(sequences);
        int offset = stringOffset;
@@ -11,7 +11,7 @@ import org.springframework.stereotype.Service;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;

 import lombok.RequiredArgsConstructor;
@@ -33,10 +33,10 @@ public class GraphicExtractorService {
                                                PDPage pdPage,
                                                int pageNumber,
                                                CleanRulings cleanRulings,
-                                                List<TextPositionSequence> textPositionSequences,
+                                                List<Word> words,
                                                boolean graphicsRaster) {

-        List<Box> characterBBoxes = getCharacterBBoxes(textPositionSequences);
+        List<Box> characterBBoxes = getCharacterBBoxes(words);
        List<Box> classifiedRulingsBoxes = getLineBBoxesOfAllClassifiedRulings(cleanRulings);

        GraphicBBDetector graphicBBDetector = new GraphicBBDetector(pdPage, true);
@@ -63,9 +63,9 @@ public class GraphicExtractorService {
    }


-    private List<Box> getCharacterBBoxes(List<TextPositionSequence> textPositionSequences) {
+    private List<Box> getCharacterBBoxes(List<Word> words) {

-        return textPositionSequences.stream()
+        return words.stream()
                .map(BoundingBox::getBBoxPdf)
                .map(Box::new)
                .collect(Collectors.toList());
@@ -5,6 +5,7 @@ import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;

 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
@@ -39,7 +40,7 @@ import org.apache.pdfbox.text.TextPosition;

 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.Getter;
 import lombok.Setter;
@@ -50,7 +51,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
 public class PDFLinesTextStripper extends PDFTextStripper {

-    private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+    private final static Set<String> DOT_LIKE_CHARACTERS = Set.of(".", "·", "•", "․", "‧", "∙", "⋅", "・", "．", "･", "…", "⸱", "﹒", "ꞏ");
+    private final List<Word> words = new ArrayList<>();
    private final List<Ruling> rulings = new ArrayList<>();
    private final List<Ruling> graphicsPath = new ArrayList<>();
    @Setter
@@ -201,7 +203,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

        try {
            if (stroke && !getGraphicsState().getStrokingColor().isPattern() && isBlack(getGraphicsState().getStrokingColor()) || //
-                    !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && isBlack(getGraphicsState().getNonStrokingColor())) {
+                !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && isBlack(getGraphicsState().getNonStrokingColor())) {
                rulings.addAll(path);
            }
        } catch (UnsupportedOperationException e) {
@@ -244,10 +246,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                direction = textPositions.get(i).getDir();
            }

-            if (!textPositionSequences.isEmpty()) {
-                previous = textPositionSequences.get(textPositionSequences.size() - 1)
+            if (!words.isEmpty()) {
+                previous = words.get(words.size() - 1)
                        .getTextPositions()
-                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
+                        .get(words.get(words.size() - 1).getTextPositions().size() - 1);
            }

            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
@@ -257,7 +259,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

            if (textPositions.get(i).getDir() != direction && startIndex != i) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                startIndex = i;
                direction = textPositions.get(i).getDir();
            }
@@ -266,7 +268,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                    words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
            }
@@ -274,53 +276,97 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                    words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
            }

-            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i)
-                    .getUnicode()
-                    .equals("\t")) && i <= textPositions.size() - 2) {
+            if (i > 0
+                && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))
+                && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {

                    // Remove false sequence ends (whitespaces)
                    if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
                        for (TextPosition t : sublist) {
-                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
+                            words.get(words.size() - 1).add(t);
                        }
                    } else {
-                        textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                        words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                    }
                }
                startIndex = i + 1;
            }
+            if (isDottedLineFollowedByWord(textPositions, i, startIndex)) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                startIndex = i;
+            }
+            if (isWordFollowedByDottedLine(textPositions, i, startIndex)) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i - 2);
+                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                startIndex = i - 2;
+            }
        }

        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1)
-                .getUnicode()
-                .equals("\u00A0") || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
+        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ")
+                                   || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0")
+                                   || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
            sublist = sublist.subList(0, sublist.size() - 1);
        }

        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
                .getUnicode()
                .equals("\t")))) {
-            if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                    .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
+            if (previous != null
+                && sublist.get(0).getYDirAdj() == previous.getYDirAdj()
+                && sublist.get(0).getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
                for (TextPosition t : sublist) {
-                    textPositionSequences.get(textPositionSequences.size() - 1).add(t);
+                    words.get(words.size() - 1).add(t);
                }
            } else {
-                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, isParagraphStart));
+                words.add(new Word(sublist, pageNumber, isParagraphStart));
            }
        }
+
        super.writeString(text);
    }


+    private boolean isWordFollowedByDottedLine(List<TextPosition> textPositions, int i, int startIndex) {
+
+        return i - startIndex >= 4 //
+               && isDot(textPositions, i) //
+               && isDot(textPositions, i - 1) //
+               && isDot(textPositions, i - 2) //
+               && alphanumeric(textPositions, i - 3);
+    }
+
+
+    private static boolean isDottedLineFollowedByWord(List<TextPosition> textPositions, int i, int startIndex) {
+
+        return i - startIndex >= 4 //
+               && alphanumeric(textPositions, i)  //
+               && isDot(textPositions, i - 1) //
+               && isDot(textPositions, i - 2) //
+               && isDot(textPositions, i - 3);
+    }
+
+
+    private static boolean isDot(List<TextPosition> textPositions, int i) {
+
+        return DOT_LIKE_CHARACTERS.contains(textPositions.get(i).getUnicode());
+    }
+
+
+    private static boolean alphanumeric(List<TextPosition> textPositions, int i) {
+
+        return Character.isAlphabetic(textPositions.get(i).getUnicode().charAt(0)) || Character.isDigit(textPositions.get(i).getUnicode().charAt(0));
+    }
+
+
    public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {

        return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
@@ -337,15 +383,16 @@ public class PDFLinesTextStripper extends PDFTextStripper {

    public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {

-        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
+        return previous != null
+               && sublist.get(0).getYDirAdj() == previous.getYDirAdj()
+               && sublist.get(0).getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
    }


    @Override
    public String getText(PDDocument doc) throws IOException {

-        textPositionSequences.clear();
+        words.clear();
        rulings.clear();
        graphicsPath.clear();
        path_x = 0.0f;
@@ -36,6 +36,9 @@ public class LayoutGridService {
        LayoutGrid layoutGrid = createLayoutGrid(document);
        Outline outline = OutlineMapper.createOutline(document);
        layoutGrid.setVisibleByDefault(layerVisibilityDefaultValue);
+
+        document.getLayoutDebugLayer().addSentenceVisualization(document.getTextBlock());
+
        if (document.getLayoutDebugLayer().isActive()) {
            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid, document.getLayoutDebugLayer()), outline);
        } else {
@@ -13,7 +13,7 @@ import org.apache.pdfbox.text.TextPosition;

 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@@ -48,7 +48,7 @@ public class MarkedContentUtils {

        return markedContentByYPosition.values()
                .stream()
-                .map(textPositions -> new TextPositionSequence(textPositions, 0, true).getBBoxPdf())
+                .map(textPositions -> new Word(textPositions, 0, true).getBBoxPdf())
                .map(t -> new Rectangle2D.Double(t.getX(), t.getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight())))
                .collect(Collectors.toList());
    }
@@ -89,7 +89,7 @@ public class MarkedContentUtils {
                    .filter(content -> content instanceof TextPosition)
                    .map(content -> (TextPosition) content)
                    .filter(content -> !content.getUnicode().equals(" "))
-                    .map(textPositions -> new TextPositionSequence(List.of(textPositions), 0, true))
+                    .map(textPositions -> new Word(List.of(textPositions), 0, true))
                    .map(BoundingBox::getBBoxPdf)
                    .collect(Collectors.toList());
        }
@@ -10,6 +10,7 @@ public final class TextNormalizationUtilities {
    public static final Pattern hyphenLineBreaks = Pattern.compile("[-~‐‒⁻−﹣゠⁓‑\\u00AD][\\r\\n]+");
    public static final Pattern linebreaks = Pattern.compile("[\\r\\n]+");
    public static final Pattern doubleWhitespaces = Pattern.compile("\\s{2,}");
+    public static final Pattern WHITESPACE_REMOVAL = Pattern.compile("\\s+");


    public String cleanString(String value) {
@@ -36,4 +37,11 @@ public final class TextNormalizationUtilities {

        return linebreaks.matcher(value).replaceAll(" ");
    }
+
+
+    public String removeAllWhitespaces(String value) {
+
+        return WHITESPACE_REMOVAL.matcher(value).replaceAll("");
+    }
+
 }
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;

 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -15,7 +16,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Union
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;

 import lombok.experimental.UtilityClass;

@@ -35,63 +36,71 @@ public class TextPositionOperations {
                    .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, X_THRESHOLD));


-    public List<TextPositionSequence> mergeAndSort(List<TextPageBlock> textBlocks) {
+    public List<Word> mergeAndSort(List<TextPageBlock> textBlocks) {

        var sequences = textBlocks.stream()
-                .flatMap(tb -> tb.getSequences()
+                .flatMap(tb -> tb.getWords()
                        .stream())
                .collect(Collectors.toSet());
        return sortUsingLineDetection(sequences);
    }


-    public List<TextPositionSequence> sort(List<TextPositionSequence> sequences) {
+    public List<Word> sort(List<Word> sequences) {

        return sortUsingLineDetection(new HashSet<>(sequences));
    }


-    private List<TextPositionSequence> sortUsingLineDetection(Set<TextPositionSequence> sequences) {
+    private List<Word> sortUsingLineDetection(Set<Word> sequences) {

        return sortLines(groupByLine(sequences));
-
    }


-    public List<TextPositionSequence> sortLines(Collection<Set<TextPositionSequence>> lines) {
+    public List<Word> sortLines(Collection<Set<Word>> lines) {

-        return lines.stream()
-                .map(TextPositionOperations::sortByXDirAdj)
-                .filter(line -> !line.isEmpty())
-                .sorted(Comparator.comparing(line -> line.get(0), COMPARATOR_DIR_ADJ))
-                .flatMap(Collection::stream)
-                .toList();
+        List<List<Word>> lineBlocks = new ArrayList<>();
+        for (Set<Word> line : lines) {
+            List<Word> sortedLine = sortByXDirAdj(line);
+            if (!sortedLine.isEmpty()) {
+                lineBlocks.add(sortedLine);
+            }
+        }
+        // need to use old sorting, since COMPARATOR_DIR_ADJ is not transitive
+        QuickSort.sort(lineBlocks, Comparator.comparing(line -> line.get(0), COMPARATOR_DIR_ADJ));
+
+        List<Word> list = new ArrayList<>();
+        for (List<Word> words : lineBlocks) {
+            list.addAll(words);
+        }
+        return list;
    }


-    private List<TextPositionSequence> sortByXDirAdj(Set<TextPositionSequence> line) {
+    private List<Word> sortByXDirAdj(Set<Word> line) {

        return line.stream()
-                .sorted(Comparator.comparing(TextPositionSequence::getXDirAdj))
+                .sorted(Comparator.comparing(Word::getXDirAdj))
                .toList();
    }


-    public Collection<Set<TextPositionSequence>> groupByLine(Set<TextPositionSequence> sequences) {
+    public Collection<Set<Word>> groupByLine(Set<Word> sequences) {

        double maxLineDistance = sequences.stream()
-                                         .map(TextPositionSequence::getBBoxDirAdj)
+                                         .map(Word::getBBoxDirAdj)
                                         .mapToDouble(RectangularShape::getHeight).average().orElse(10) * MAX_LINE_HEIGHT_FACTOR;
        double maxXGap = sequences.stream()
-                                 .map(TextPositionSequence::getBBoxDirAdj)
+                                 .map(Word::getBBoxDirAdj)
                                 .mapToDouble(RectangularShape::getWidth).average().orElse(75) * MAX_WORD_DISTANCE_FACTOR;

-        UnionFind<TextPositionSequence> unionFind = new UnionFind<>(sequences);
+        UnionFind<Word> unionFind = new UnionFind<>(sequences);

-        for (TextPositionSequence sequence : sequences) {
-            for (TextPositionSequence sequence2 : sequences) {
+        for (Word sequence : sequences) {
+            for (Word sequence2 : sequences) {

-                if (sequence.equals(sequence2) || unionFind.inSameSet(sequence, sequence2)) {
+                if (sequence.equals(sequence2)) { // || unionFind.inSameSet(sequence, sequence2)) doing this is actually slower than not doing it
                    continue;
                }

@@ -135,10 +144,10 @@ public class TextPositionOperations {
    }


-    public List<TextPositionSequence> merge(List<TextPageBlock> textBlocks) {
+    public List<Word> merge(List<TextPageBlock> textBlocks) {

        return textBlocks.stream()
-                .map(TextPageBlock::getSequences)
+                .map(TextPageBlock::getWords)
                .flatMap(Collection::stream)
                .collect(Collectors.toList());
    }
@@ -5,9 +5,11 @@ import java.awt.geom.AffineTransform;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
+import java.text.BreakIterator;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -19,10 +21,13 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Bound
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
@@ -53,14 +58,14 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
    Map<Integer, AtomicInteger> outlineObjectsWithoutPointsPerPage = new HashMap<>();


-    public void addTextVisualizations(List<TextPositionSequence> textPositionSequences, int pageNumber) {
+    public void addTextVisualizations(List<Word> words, int pageNumber) {

        if (!active) {
            return;
        }
        VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(pageNumber, this.words);
        visualizationsOnPage.getColoredRectangles()
-                .addAll(textPositionSequences.stream()
+                .addAll(words.stream()
                                .map(BoundingBox::getBBoxPdf)
                                .map(rect -> new ColoredRectangle(rect, WORDS_COLOR, 1))
                                .toList());
@@ -76,7 +81,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
        visualizationsOnPage.getColoredLines()
                .addAll(cleanRulings.buildAll()
                                .stream()
-                                .map(ruling -> new ColoredLine(ruling, decideOnRulingColor(ruling), 0.5f))
+                                .map(ruling -> new ColoredLine(ruling, decideOnRulingColor(ruling), LINE_WIDTH))
                                .toList());
    }

@@ -89,11 +94,34 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
        VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(pageNumber, this.rulings);
        visualizationsOnPage.getColoredLines()
                .addAll(rulings.stream()
-                                .map(ruling -> new ColoredLine(ruling, decideOnRulingColor(ruling), 0.5f))
+                                .map(ruling -> new ColoredLine(ruling, decideOnRulingColor(ruling), LINE_WIDTH))
                                .toList());
    }


+    public void addSentenceVisualization(TextBlock textBlock) {
+
+        if (!active) {
+            return;
+        }
+        AtomicInteger rotatingColorIdx = new AtomicInteger(0);
+        String text = textBlock.getSearchText();
+        BreakIterator sentenceIterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
+        sentenceIterator.setText(text);
+        int lastIdx = 0;
+        while (sentenceIterator.next() != BreakIterator.DONE) {
+            TextRange sentenceRange = new TextRange(lastIdx + textBlock.getTextRange().start(), sentenceIterator.current() + textBlock.getTextRange().start());
+            lastIdx = sentenceIterator.current();
+            Color color = getRotatingColor(rotatingColorIdx);
+            textBlock.getPositionsPerPage(sentenceRange)
+                    .forEach((page, bboxes) -> getOrCreateVisualizationsOnPage(page.getNumber(), this.sentences).getColoredRectangles()
+                            .addAll(bboxes.stream()
+                                            .map(bbox -> new ColoredRectangle(bbox, color, 1))
+                                            .toList()));
+        }
+    }
+
+
    private Color decideOnRulingColor(Ruling ruling) {

        return switch (ruling.getClassification()) {
@@ -155,12 +183,12 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
        visualizationsOnPage.getColoredRectangles()
                .addAll(lines.stream()
                                .map(BoundingBox::getBBoxPdf)
-                                .map(line -> new ColoredRectangle(line, LINES_COLOR, 0.5f))
+                                .map(line -> new ColoredRectangle(line, LINES_COLOR, LINE_WIDTH))
                                .toList());
    }


-    public void addLineVisualizationsFromNestedTextPosition(Collection<Set<TextPositionSequence>> lines, int pageNumber) {
+    public void addLineVisualizationsFromNestedTextPosition(Collection<Set<Word>> lines, int pageNumber) {

        if (!active) {
            return;
@@ -171,7 +199,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
                                .map(line -> line.stream()
                                        .map(BoundingBox::getBBoxPdf)
                                        .collect(RectangleTransformations.collectBBox()))
-                                .map(line -> new ColoredRectangle(line, LINES_COLOR, 0.5f))
+                                .map(line -> new ColoredRectangle(line, LINES_COLOR, LINE_WIDTH))
                                .toList());

    }
@@ -248,7 +276,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
                .map(Line::getCharacters)
                .flatMap(Collection::stream)
                .forEach(character -> {
-                    Color color = ROTATING_CHARACTER_COLOR.get(index.getAndIncrement() % ROTATING_CHARACTER_COLOR.size());
+                    Color color = getRotatingColor(index);
                    Rectangle2D charBBox = character.getTextPosition().getBBoxPdf();
                    characterVisualizations.getColoredRectangles().add(new ColoredRectangle(charBBox, color, 1));
                    character.getNeighbors()
@@ -263,6 +291,31 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
    }


+    public void addTocPages(List<Word> numbers, int page) {
+
+        if (!active) {
+            return;
+        }
+
+        VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(page, this.tocPages);
+        visualizationsOnPage.getColoredRectangles()
+                .addAll(numbers.stream()
+                                .map(BoundingBox::getBBoxPdf)
+                                .map(line -> new ColoredRectangle(line, LINES_COLOR, LINE_WIDTH))
+                                .toList());
+        visualizationsOnPage.getColoredRectangles()
+                .add(new ColoredRectangle(numbers.stream()
+                                                  .map(BoundingBox::getBBoxPdf)
+                                                  .collect(RectangleTransformations.collectBBox()), LINES_COLOR, LINE_WIDTH));
+    }
+
+
+    private static Color getRotatingColor(AtomicInteger index) {
+
+        return ROTATING_CHARACTER_COLOR.get(index.getAndIncrement() % ROTATING_CHARACTER_COLOR.size());
+    }
+
+
    public void addOutlineObjects(List<OutlineObject> outlineObjects, PageInformation pageInformation) {

        if (!active) {
@@ -299,4 +352,13 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
        visualizationsOnPage.getPlacedTexts().add(PlacedText.textFacingUp(outlineObject.getTitle(), textPoint, 10, outlineObject.isFound() ? Color.BLACK : Color.RED, FONT));
    }

+
+    public void addListIdentifiers(List<ListIdentifier> listIdentifiers) {
+
+        for (ListIdentifier listIdentifier : listIdentifiers) {
+            getOrCreateVisualizationsOnPage(listIdentifier.getPage(), this.listIdentifiers).getColoredRectangles()
+                    .add(new ColoredRectangle(listIdentifier.getWord().getBBoxPdf(), WORDS_COLOR, LINE_WIDTH));
+        }
+    }
+
 }
@@ -0,0 +1,70 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.awt.geom.Rectangle2D;
+
+import org.junit.jupiter.api.Test;
+
+class BoundingBoxTest {
+
+    @Test
+    void testHorizontalDistance_NoOverlap() {
+
+        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
+        ConcreteBoundingBox box2 = new ConcreteBoundingBox(20, 0, 10, 10);
+
+        assertEquals(10, box1.horizontalDistance(box2));
+    }
+
+
+    @Test
+    void testHorizontalDistance_Overlap() {
+
+        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
+        ConcreteBoundingBox box2 = new ConcreteBoundingBox(5, 0, 10, 10);
+
+        assertEquals(0, box1.horizontalDistance(box2));
+    }
+
+
+    @Test
+    void testVerticalDistance_NoOverlap() {
+
+        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
+        ConcreteBoundingBox box2 = new ConcreteBoundingBox(0, 20, 10, 10);
+
+        assertEquals(10, box1.verticalDistance(box2));
+    }
+
+
+    @Test
+    void testVerticalDistance_Overlap() {
+
+        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
+        ConcreteBoundingBox box2 = new ConcreteBoundingBox(0, 5, 10, 10);
+
+        assertEquals(0, box1.verticalDistance(box2));
+    }
+
+
+    @Test
+    void testVerticalDistance_PartialOverlap() {
+
+        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
+        ConcreteBoundingBox box2 = new ConcreteBoundingBox(0, 8, 10, 10);
+
+        assertEquals(0, box1.verticalDistance(box2));
+    }
+
+
+    @Test
+    void testHorizontalDistance_PartialOverlap() {
+
+        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
+        ConcreteBoundingBox box2 = new ConcreteBoundingBox(8, 0, 10, 10);
+
+        assertEquals(0, box1.horizontalDistance(box2));
+    }
+
+}
@@ -0,0 +1,12 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+
+class ConcreteBoundingBox extends BoundingBox {
+
+    ConcreteBoundingBox(double x, double y, double width, double height) {
+
+        this.bBox = new Rectangle2D.Double(x, y, width, height);
+    }
+
+}
@@ -37,7 +37,7 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
    @Disabled
    public void testLayoutParserEndToEnd() {

-        String filePath = "/home/kschuettler/Dokumente/TestFiles/NER Dataset/Syngenta prod/77c680315c31d403d2e023be023b2087.PREVIEW.pdf";
+        String filePath = "/home/kschuettler/Dokumente/LayoutparsingEvaluation/VV-340050.pdf";

        runForFile(filePath);
    }
@@ -79,7 +79,7 @@ public class OutlineDetectionTest extends AbstractTest {
        var documentFile = new ClassPathResource(fileName).getFile();

        long start = System.currentTimeMillis();
-        ClassificationDocument classificationDocument = parseLayout(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
+        ClassificationDocument classificationDocument = parseLayout(fileName, LayoutParsingType.DOCUMINE_OLD);
        Document document = buildGraph(fileName, classificationDocument);
        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
        OutlineObjectTree outlineObjectTree = classificationDocument.getOutlineObjectTree();
@@ -102,7 +102,7 @@ public class OutlineDetectionTest extends AbstractTest {

        TableOfContents tableOfContents = classificationDocument.getTableOfContents();

-        assertEquals(tableOfContents.getMainSections().size(), 10);
+        assertEquals(tableOfContents.getMainSections().size(), 9);
        assertEquals(tableOfContents.getMainSections().subList(1, 9)
                             .stream()
                             .map(tableOfContentItem -> sanitizeString(tableOfContentItem.getHeadline().toString()))
@@ -135,7 +135,7 @@ public class OutlineDetectionTest extends AbstractTest {

        List<SemanticNode> childrenOfTypeSectionOrSuperSection = document.getChildrenOfTypeSectionOrSuperSection();

-        assertEquals(childrenOfTypeSectionOrSuperSection.size(), 10);
+        assertEquals(childrenOfTypeSectionOrSuperSection.size(), 9);
        assertEquals(childrenOfTypeSectionOrSuperSection.subList(1, 9)
                             .stream()
                             .map(section -> sanitizeString(section.getHeadline().getLeafTextBlock().toString()))
@@ -46,8 +46,8 @@ public class SimplifiedTextServiceTest
        Document document = buildGraph(file);
        SimplifiedText simplifiedText = simplifiedSectionTextService.toSimplifiedText(document);
        List<SimplifiedSectionText> sectionTexts = simplifiedText.getSectionTexts();
-        assertThat(sectionTexts.stream().filter(section -> section.getText().equals(footerExample)).collect(Collectors.toList()).size()).isGreaterThan(0);
-        assertThat(sectionTexts.stream().filter(section -> section.getText().equals(headerExample)).collect(Collectors.toList()).size()).isGreaterThan(0);
+        assertThat(sectionTexts.stream().filter(section -> section.getText().contains(footerExample)).toList().size()).isGreaterThan(0);
+        assertThat(sectionTexts.stream().filter(section -> section.getText().contains(headerExample)).toList().size()).isGreaterThan(0);



@@ -26,7 +26,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 import com.knecon.fforesight.service.layoutparser.server.PDFNetInitializer;
@@ -51,7 +51,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
 public class DocumentReadingOrderTest extends BuildDocumentTest {

-    private static final boolean DRAW_DIR_ADJ_COORDS = true;
+    private static final boolean DRAW_DIR_ADJ_COORDS = false;
    public static final List<LayoutParsingType> LAYOUT_PARSING_TYPES = List.of(LayoutParsingType.DOCUMINE,
                                                                               LayoutParsingType.DOCUMINE_OLD,
                                                                               LayoutParsingType.REDACT_MANAGER,
@@ -82,7 +82,7 @@ public class DocumentReadingOrderTest extends BuildDocumentTest {
    @Disabled
    public void drawDirAdjForFile() {

-        String pdfFile = "/home/kschuettler/Dokumente/Ticket Related/RED-9974/026dc94b019bc2348a4c54f0c6c4516f.ORIGIN.pdf";
+        String pdfFile = "/home/kschuettler/Dokumente/TestFiles/OCR/VV-331340/VV-331340_OCRED_first15.pdf";

        ClassificationDocument classificationDocument = parseLayout(pdfFile, LayoutParsingType.DOCUMINE_OLD);

@@ -387,7 +387,7 @@ public class DocumentReadingOrderTest extends BuildDocumentTest {
                for (AbstractPageBlock abstractBlock : classificationDocumentPage.getTextBlocks()) {

                    if (abstractBlock instanceof TextPageBlock textBlock) {
-                        for (TextPositionSequence sequence : TextPositionOperations.mergeAndSort(List.of(textBlock))) {
+                        for (Word sequence : TextPositionOperations.mergeAndSort(List.of(textBlock))) {

                            float stringWidth;
                            try {
@@ -76,9 +76,10 @@ class TextRangeTest {
        assertEquals(List.of(new TextRange(10, 40), new TextRange(40, 100)), startTextRange.split(List.of(40)));
        assertEquals(1, startTextRange.split(Collections.emptyList()).size());
        assertEquals(1, startTextRange.split(List.of(startTextRange.start())).size());
+        assertEquals(1, startTextRange.split(List.of(100)).size());
        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(Collections.singletonList(0)));
-        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(Collections.singletonList(100)));
-        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(List.of(12, 40, 100)));
+        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(Collections.singletonList(101)));
+        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(List.of(12, 40, 101)));
    }

 }
@@ -32,7 +32,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.textbloc
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
@@ -105,9 +105,9 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        List<PageContents> textPositionPerPage = PageContentExtractor.getSortedPageContents(fileName);
        var textPositions = textPositionPerPage.stream()
-                .flatMap(t -> t.getSortedTextPositionSequences()
+                .flatMap(t -> t.getSortedWords()
                        .stream()
-                        .map(TextPositionSequence::toString))
+                        .map(Word::toString))
                .collect(Collectors.joining(" "));
        assertThat(textPositions.contains(textToSearch)).isFalse();

@@ -117,7 +117,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                           .get(0).getTextBlocks().size()).isEqualTo(3);
        assertThat(classificationDocument.getHeaders()
                           .get(0).getTextBlocks()
-                           .get(0).getSequences().size()).isEqualTo(8);
+                           .get(0).getWords().size()).isEqualTo(8);
        assertThat(classificationDocument.getHeaders()
                           .get(0).getTextBlocks()
                           .get(0).toString()).contains(textToSearch);
@@ -36,7 +36,7 @@ class GapAcrossLinesDetectionServiceTest {
        System.out.println("start column detection");
        start = System.currentTimeMillis();
        for (PageInformation pageInformation : pageInformations) {
-            GapInformation gapInformation = GapDetectionService.findGapsInLines(pageInformation.getPageContents().getSortedTextPositionSequences(),  pageInformation.getMainBodyTextFrame());
+            GapInformation gapInformation = GapDetectionService.findGapsInLines(pageInformation.getPageContents().getSortedWords(), pageInformation.getMainBodyTextFrame());
            columnsPerPage.add(GapsAcrossLinesService.detectXGapsAcrossLines(gapInformation, pageInformation.getMainBodyTextFrame()));
        }
        System.out.printf("Finished column detection in %d ms%n", System.currentTimeMillis() - start);
@@ -10,7 +10,7 @@ import java.util.stream.Collectors;
 import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.services.InvisibleTableDetectionService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
@@ -34,18 +34,18 @@ class InvisibleTableDetectionServiceTest {
                .collect(Collectors.toList());

        int pageNumber = 1;
-        Rectangle2D tableBBox = pageContents.get(0).getPageContents().getSortedTextPositionSequences().subList(45, 152)
+        Rectangle2D tableBBox = pageContents.get(0).getPageContents().getSortedWords().subList(45, 152)
                .stream()
-                .map(TextPositionSequence::getBBox)
+                .map(Word::getBBox)
                .map(this::mirrorY)
                .collect(RectangleTransformations.collectBBox());

-        List<TextPositionSequence> textPositionSequences = pageContents.get(0).getPageContents().getSortedTextPositionSequences()
+        List<Word> words = pageContents.get(0).getPageContents().getSortedWords()
                .stream()
                .filter(textPositionSequence -> tableBBox.contains(mirrorY(textPositionSequence.getBBox())))
                .toList();

-        var table = InvisibleTableDetectionService.detectTable(textPositionSequences, tableBBox);
+        var table = InvisibleTableDetectionService.detectTable(words, tableBBox);

        PdfDraw.drawRectanglesPerPage(fileName,
                                      List.of(table.stream()
@@ -6,7 +6,7 @@ import java.util.List;
 import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

@@ -25,9 +25,9 @@ class PageContentExtractorTest {

        PdfDraw.drawRectanglesPerPageNumberedByLine(fileName,
                textPositionPerPage.stream()
-                        .map(t -> t.getSortedTextPositionSequences()
+                        .map(t -> t.getSortedWords()
                                .stream()
-                                .map(TextPositionSequence::getBBoxPdf)
+                                .map(Word::getBBoxPdf)
                                .map(List::of)
                                .toList())
                        .toList(), tmpFileName);
@@ -11,7 +11,7 @@ import org.junit.jupiter.api.Test;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TextRulingsClassifier;
@@ -32,16 +32,16 @@ public class RulingsClassifierTest {
        for (PageContents pageContent : pageContents) {
            CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(Collections.emptyList(), pageContent.getRulings());
            RectangularIntersectionFinder.find(cleanRulings.getHorizontals(), cleanRulings.getVerticals());
-            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(pageContent.getSortedTextPositionSequences(), cleanRulings);
+            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(pageContent.getSortedWords(), cleanRulings);

-            assertTrue(pageContent.getSortedTextPositionSequences()
+            assertTrue(pageContent.getSortedWords()
                               .stream()
                               .filter(word -> word.toString().equals("Underlined"))
-                               .allMatch(TextPositionSequence::isUnderline));
-            assertTrue(pageContent.getSortedTextPositionSequences()
+                               .allMatch(Word::isUnderline));
+            assertTrue(pageContent.getSortedWords()
                               .stream()
                               .filter(word -> word.toString().equals("Striketrough"))
-                               .allMatch(TextPositionSequence::isStrikethrough));
+                               .allMatch(Word::isStrikethrough));

            assertEquals(4,
                         cleanRulings.buildAll()
@@ -70,7 +70,7 @@ public class RulingsClassifierTest {
        for (PageContents pageContent : pageContents) {
            CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(Collections.emptyList(), pageContent.getRulings());
            RectangularIntersectionFinder.find(cleanRulings.getHorizontals(), cleanRulings.getVerticals());
-            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(pageContent.getSortedTextPositionSequences(), cleanRulings);
+            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(pageContent.getSortedWords(), cleanRulings);

            assertEquals(30, cleanRulings.getHorizontals().size());
            assertEquals(30, cleanRulings.getTableLines().getHorizontals().size());
@@ -1,8 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

-import java.awt.geom.Rectangle2D;
 import java.io.File;
-import java.io.FileOutputStream;
 import java.nio.file.Path;
 import java.util.Map;

@@ -12,27 +10,11 @@ import org.springframework.core.io.ClassPathResource;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.viewerdoc.model.Standard14EmbeddableFont;
-import com.pdftron.common.Matrix2D;
-import com.pdftron.pdf.ColorPt;
-import com.pdftron.pdf.ColorSpace;
-import com.pdftron.pdf.Element;
-import com.pdftron.pdf.ElementBuilder;
-import com.pdftron.pdf.ElementWriter;
-import com.pdftron.pdf.Font;
-import com.pdftron.pdf.GState;
-import com.pdftron.pdf.PDFDoc;
-import com.pdftron.pdf.Page;
-import com.pdftron.sdf.SDFDoc;

 import lombok.SneakyThrows;

@@ -55,7 +55,11 @@ public record LayerIdentifier(String name, String markedContentName) {
    public static final LayerIdentifier NEIGHBOURS = new LayerIdentifier("Neighbours", "NEIGHBOURS");
    public static final LayerIdentifier CHARACTERS = new LayerIdentifier("Characters", "CHARACTERS");
    public static final LayerIdentifier OUTLINE_OBJECTS = new LayerIdentifier("Outline Positions", "OUTLINE_OBJECTS");
+    public static final LayerIdentifier SENTENCES = new LayerIdentifier("Sentences", "SENTENCES");
+    public static final LayerIdentifier TOC_PAGES = new LayerIdentifier("TOC pages", "TOC_PAGES");
+    public static final LayerIdentifier LIST_IDENTIFIERS = new LayerIdentifier("List identifiers", "LIST_IDENTIFIERS");

+    // Visual layout parser
    public static final LayerIdentifier KNECON_VISUAL_PARSING = new LayerIdentifier("Visual Layout Parser", "VISUAL_PARSING");

    //ocr
@@ -18,6 +18,8 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {

    protected static final Standard14EmbeddableFont FONT = Standard14EmbeddableFont.helvetica();

+    public static final float LINE_WIDTH = 0.5f;
+
    protected static final Color WORDS_COLOR = new Color(68, 84, 147);
    protected static final Color LINES_COLOR = new Color(152, 45, 179);
    protected static final Color ZONES_COLOR = new Color(131, 38, 38);
@@ -44,9 +46,9 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
                                                                          new Color(0, 188, 212),
                                                                          new Color(121, 85, 72));

-    protected final Visualizations words = Visualizations.builder().layer(LayerIdentifier.WORDS).visibleByDefault(true).build();
+    protected final Visualizations words = Visualizations.builder().layer(LayerIdentifier.WORDS).build();
    protected final Visualizations lines = Visualizations.builder().layer(LayerIdentifier.LINES).build();
-    protected final Visualizations zones = Visualizations.builder().layer(LayerIdentifier.ZONES).build();
+    protected final Visualizations zones = Visualizations.builder().layer(LayerIdentifier.ZONES).visibleByDefault(true).build();
    protected final Visualizations mainBody = Visualizations.builder().layer(LayerIdentifier.MAIN_BODY).build();
    protected final Visualizations clean_rulings = Visualizations.builder().layer(LayerIdentifier.CLEAN_RULINGS).build();
    protected final Visualizations rulings = Visualizations.builder().layer(LayerIdentifier.RULINGS).build();
@@ -55,6 +57,9 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
    protected final Visualizations neighbours = Visualizations.builder().layer(LayerIdentifier.NEIGHBOURS).build();
    protected final Visualizations characters = Visualizations.builder().layer(LayerIdentifier.CHARACTERS).build();
    protected final Visualizations outlineObjects = Visualizations.builder().layer(LayerIdentifier.OUTLINE_OBJECTS).build();
+    protected final Visualizations sentences = Visualizations.builder().layer(LayerIdentifier.SENTENCES).build();
+    protected final Visualizations tocPages = Visualizations.builder().layer(LayerIdentifier.TOC_PAGES).build();
+    protected final Visualizations listIdentifiers = Visualizations.builder().layer(LayerIdentifier.LIST_IDENTIFIERS).build();


    public List<Visualizations> getVisualizations() {
@@ -63,14 +68,18 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
                       neighbours,//
                       words, //
                       lines, //
+                       sentences, //
                       zones, //
                       rulings, //
                       clean_rulings, //
                       cells, //
                       mainBody, //
                       markedContent, //
-                       outlineObjects //
+                       outlineObjects, //
+                       tocPages, //
+                       listIdentifiers //
        );
    }

+
 }
@@ -25,10 +25,11 @@ public class OcrDebugLayerConfig extends AbstractLayerGroup {

    protected final Visualizations debugText = Visualizations.builder().layer(LayerIdentifier.KNECON_OCR_TEXT_DEBUG).visibleByDefault(true).build();
    protected final Visualizations tableLines = Visualizations.builder().layer(LayerIdentifier.KNECON_OCR_LINE_DEBUG).visibleByDefault(true).build();
-    protected final Visualizations overlappedText = Visualizations.builder().layer(LayerIdentifier.KNECON_OCR_OVERLAPPED_TEXT).visibleByDefault(true).build();
+    protected final Visualizations overlappedText = Visualizations.builder().layer(LayerIdentifier.KNECON_OCR_OVERLAPPED_TEXT).visibleByDefault(false).build();
    protected final Visualizations debugBBox = Visualizations.builder().layer(LayerIdentifier.KNECON_OCR_BBOX_DEBUG).visibleByDefault(false).build();


+
    @Override
    public List<Visualizations> getVisualizations() {

@@ -36,10 +37,4 @@ public class OcrDebugLayerConfig extends AbstractLayerGroup {
    }


-    @Override
-    public boolean isVisibleByDefault() {
-
-        return true;
-    }
-
 }
@@ -68,10 +68,10 @@ public class OutlineUtility {
    public static void deleteExistingOutline(PDFDoc doc) {

        Bookmark firstBookmark = doc.getFirstBookmark();
-//        while (firstBookmark != null && firstBookmark.isValid()) {
+        while (firstBookmark != null && firstBookmark.isValid()) {
            firstBookmark.delete();
            firstBookmark = doc.getFirstBookmark();
-//        }
+        }

    }
Author	SHA1	Message	Date
Kilian Schüttler	9297d6b83d	Merge branch 'RED-10270' into 'release/0.188.x' RED-10270: fix NumberFormatException See merge request fforesight/layout-parser!249	2024-10-24 17:14:50 +02:00
Kilian Schuettler	230154a2a8	RED-10270: fix NumberFormatException	2024-10-24 10:59:30 +02:00
Kilian Schüttler	2219519a2b	Merge branch 'RED-10127' into 'main' RED-10127: rename TextPositionSequence to Word See merge request fforesight/layout-parser!244	2024-10-18 12:20:15 +02:00
Kilian Schüttler	af05218e37	RED-10127: rename TextPositionSequence to Word	2024-10-18 12:20:15 +02:00
Kilian Schüttler	736f531df3	Merge branch 'hotfix' into 'main' Hotfix See merge request fforesight/layout-parser!243	2024-10-18 12:12:15 +02:00
Kilian Schüttler	c64445d54b	Hotfix	2024-10-18 12:12:15 +02:00
Kilian Schüttler	af29233b10	Merge branch 'feature/RED-10127' into 'main' RED-10127: add more units See merge request fforesight/layout-parser!242	2024-10-15 09:57:21 +02:00
Kilian Schuettler	5f04b45554	RED-10127: add more units	2024-10-15 09:47:39 +02:00
Kilian Schüttler	6c41533f0b	Merge branch 'feature/RED-10127' into 'main' RED-10127: improve list classification See merge request fforesight/layout-parser!240	2024-10-14 17:34:33 +02:00
Kilian Schuettler	9d2596e5ef	RED-10127: improve list classification * add one more format to list identification * add 'ppb' to known units * special case for headlines continuing with 14C after the identifier (quite often in some specific files)	2024-10-14 17:21:44 +02:00
Kilian Schüttler	e7b01161ac	Merge branch 'feature/RED-10127' into 'main' RED-10127: add list classification See merge request fforesight/layout-parser!237	2024-10-10 10:50:10 +02:00
Kilian Schüttler	7b073eb4f3	RED-10127: add list classification	2024-10-10 10:50:10 +02:00
Dominique Eifländer	4b0c041d84	Merge branch 'feature/RED-10127' into 'main' RED-10127: improve headline detection See merge request fforesight/layout-parser!235	2024-10-09 08:48:48 +02:00
Kilian Schüttler	6c7442ac6d	RED-10127: improve headline detection	2024-10-09 08:48:48 +02:00