Merge branch 'RED-9975' into 'main'

Red 9975: fix outline detection

See merge request fforesight/layout-parser!206

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -51,10 +51,6 @@ allprojects {
         }
     }
 
-    pmd {
-        setConsoleOutput(true)
-    }
-
     publishing {
         publications {
             create<MavenPublication>(name) {

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -21,5 +21,14 @@ public class SimplifiedText {
     @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
     @Builder.Default
     private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+    @Schema(description = "A list of the main section numbers ")
+    @Builder.Default
+    private List<String> mainSectionNumbers = new ArrayList<>();
+    @Schema(description = "A list of the header section numbers ")
+    @Builder.Default
+    private List<String> headerSectionNumbers = new ArrayList<>();
+    @Schema(description = "A list of the footer section numbers ")
+    @Builder.Default
+    private List<String> footerSectionNumbers = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,6 +2,9 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
-    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE_PREFIX = "layout_parsing_request";
+    public static final String LAYOUT_PARSING_REQUEST_EXCHANGE = "layout_parsing_request_exchange";
+    public static final String LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX = "layout_parsing_response";
+    public static final String LAYOUT_PARSING_RESPONSE_EXCHANGE = "layout_parsing_response_exchange";
+    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_error";
 }

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -16,7 +16,7 @@ dependencies {
         exclude("org.springframework.boot", "spring-boot-starter-security")
         exclude("org.springframework.boot", "spring-boot-starter-validation")
     }
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.29.0")
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
 
     implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
@@ -25,9 +25,6 @@ dependencies {
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
     implementation("org.jgrapht:jgrapht-core:1.5.2")
-    implementation("org.apache.pdfbox:jbig2-imageio:3.0.4")
-    implementation("com.github.jai-imageio:jai-imageio-core:1.4.0")
-    implementation("com.github.jai-imageio:jai-imageio-jpeg2000:1.4.0")
     implementation("org.tinspin:tinspin-indexes:2.1.3")
     implementation("org.commonmark:commonmark:0.22.0")
     implementation("org.commonmark:commonmark-ext-gfm-tables:0.22.0")

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -2,13 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor;
 
 import static java.lang.String.format;
 
-import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -26,10 +25,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.No
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.MarkdownMapper;
+import com.knecon.fforesight.service.layoutparser.processor.markdown.MarkdownMapper;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -44,7 +40,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableO
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
@@ -52,6 +48,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
@@ -61,6 +58,9 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClarifyndClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.Box;
 import com.knecon.fforesight.service.layoutparser.processor.services.graphics.GraphicExtractorService;
@@ -92,7 +92,10 @@ public class LayoutParsingPipeline {
     CvTableParsingAdapter cvTableParsingAdapter;
     LayoutParsingStorageService layoutParsingStorageService;
     SectionsBuilderService sectionsBuilderService;
+    RedactManagerClassificationService redactManagerClassificationService;
+    DocuMineClassificationService docuMineClassificationService;
     SimplifiedSectionTextService simplifiedSectionTextService;
+    BodyTextFrameService bodyTextFrameService;
     RulingCleaningService rulingCleaningService;
     TableExtractionService tableExtractionService;
     DocuMineBlockificationService docuMineBlockificationService;
@@ -102,12 +105,12 @@ public class LayoutParsingPipeline {
     LayoutGridService layoutGridService;
     ObservationRegistry observationRegistry;
     VisualLayoutParsingAdapter visualLayoutParsingAdapter;
+    ClarifyndClassificationService clarifyndClassificationService;
     GraphicExtractorService graphicExtractorService;
     OutlineExtractorService outlineExtractorService;
     OutlineValidationService outlineValidationService;
     TOCEnrichmentService tocEnrichmentService;
     LayoutparserSettings settings;
-    ClassificationService classificationService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -140,7 +143,7 @@ public class LayoutParsingPipeline {
 
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
 
-        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false);
+        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, layoutParsingRequest.visualLayoutParsingFileId().isPresent());
 
         log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
 
@@ -270,10 +273,7 @@ public class LayoutParsingPipeline {
             stripper.setEndPage(pageNumber);
             stripper.setPdpage(pdPage);
             stripper.getText(originDocument);
-            List<Word> words = stripper.getWords();
-
-//            rotateDirAdjExactly(words, pdPage); // works really well for many highly rotated documents (e.g. VV-331340.pdf), but it decreases the headline performance by 1.3%, so I am leaving it out for now
-
+            List<TextPositionSequence> words = stripper.getTextPositionSequences();
             if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
                 var lines = TextPositionOperations.groupByLine(new HashSet<>(words));
                 classificationDocument.getLayoutDebugLayer().addLineVisualizationsFromNestedTextPosition(lines, pageNumber);
@@ -296,7 +296,7 @@ public class LayoutParsingPipeline {
 
             TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(words, cleanRulings);
 
-            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getWords(), false);
+            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getTextPositionSequences(), false);
 
             pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>())
                     .addAll(graphics.stream()
@@ -309,7 +309,7 @@ public class LayoutParsingPipeline {
 
             ClassificationPage classificationPage = switch (layoutParsingType) {
                 case REDACT_MANAGER_OLD ->
-                        redactManagerBlockificationService.blockify(stripper.getWords(), cleanRulings, classificationDocument.getLayoutDebugLayer());
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings, classificationDocument.getLayoutDebugLayer());
                 case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
                 case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
                         docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
@@ -367,9 +367,33 @@ public class LayoutParsingPipeline {
 
         originDocument.close();
 
-        classificationService.classify(classificationDocument, layoutParsingType, identifier);
+        log.info("Calculating BodyTextFrame for {}", identifier);
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+        for (ClassificationPage page : classificationDocument.getPages()) {
+            classificationDocument.getLayoutDebugLayer().addCleanRulingVisualization(page.getCleanRulings(), page.getPageNumber());
+        }
+        log.info("Classify TextBlocks for {}", identifier);
+        switch (layoutParsingType) {
+            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
+                    redactManagerClassificationService.classifyDocument(classificationDocument);
+            case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
+            case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
+        }
 
-        TableOfContents tableOfContents = outlineValidationService.createToC(classificationDocument);
+        if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
+            for (ClassificationPage page : classificationDocument.getPages()) {
+                docuMineBlockificationService.mergeblocks(page, page.getCleanRulings().withoutTextRulings(), 0, 10);
+            }
+        }
+
+        List<TextPageBlock> headlines = classificationDocument.getPages()
+                .stream()
+                .flatMap(classificationPage -> classificationPage.getTextBlocks()
+                        .stream()
+                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
+                        .map(tb -> (TextPageBlock) tb))
+                .toList();
+        TableOfContents tableOfContents = outlineValidationService.createToC(headlines);
         classificationDocument.setTableOfContents(tableOfContents);
 
         log.info("Building Sections for {}", identifier);
@@ -383,32 +407,6 @@ public class LayoutParsingPipeline {
     }
 
 
-    private static void rotateDirAdjExactly(List<Word> words, PDPage pdPage) {
-
-        for (TextDirection dir : TextDirection.values()) {
-
-            double averageRotation = words.stream()
-                    .map(Word::getTextPositions)
-                    .flatMap(Collection::stream)
-                    .filter(pos -> pos.getDir().equals(dir))
-                    .mapToDouble(RedTextPosition::getExactDir).average().orElse(0);
-
-            if (averageRotation == 0) {
-                continue;
-            }
-
-            AffineTransform rotateInstance = AffineTransform.getRotateInstance(averageRotation, pdPage.getMediaBox().getWidth() / 2, pdPage.getMediaBox().getHeight() / 2);
-
-            for (Word word : words) {
-                if (!dir.equals(word.getDir())) {
-                    continue;
-                }
-                word.transform(rotateInstance);
-            }
-        }
-    }
-
-
     private void addNumberOfPagesToTrace(int numberOfPages, long size) {
 
         if (observationRegistry.getCurrentObservation() != null) {
@@ -450,10 +448,10 @@ public class LayoutParsingPipeline {
         // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
         for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getWords() == null) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
                     continue;
                 }
-                for (Word word : ((TextPageBlock) textBlock).getWords()) {
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
                     classificationPage.getTextHeightCounter().add(word.getTextHeight());
                     classificationPage.getFontCounter().add(word.getFont());
                     classificationPage.getFontSizeCounter().add(word.getFontSize());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/DocstrumSegmentationService.java

@@ -18,7 +18,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.Zon
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
 
 import lombok.RequiredArgsConstructor;
@@ -35,7 +35,7 @@ public class DocstrumSegmentationService {
     private final ReadingOrderService readingOrderService;
 
 
-    public List<Zone> segmentPage(List<Word> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
+    public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
 
         EnumMap<TextDirection, Integer> directionCounts = new EnumMap<>(TextDirection.class);
 
@@ -78,11 +78,11 @@ public class DocstrumSegmentationService {
     }
 
 
-    private List<Zone> computeZones(List<Word> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
+    private List<Zone> computeZones(List<TextPositionSequence> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
 
         List<RedTextPosition> positions = textPositions.stream()
                 .filter(t -> t.getDir() == direction)
-                .map(Word::getTextPositions)
+                .map(TextPositionSequence::getTextPositions)
                 .flatMap(List::stream)
                 .toList();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/BoundingBox.java

@@ -225,31 +225,33 @@ public abstract class BoundingBox {
 
     public double horizontalDistance(BoundingBox other) {
 
-        double rect1Right = getMaxX();
-        double rect1Left = getMinX();
-        double rect2Right = other.getMaxX();
-        double rect2Left = other.getMinX();
-
-        if (rect1Left > rect2Right || rect2Left > rect1Right) {
-            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        Rectangle2D left;
+        Rectangle2D right;
+        if (this.leftOf(other)) {
+            left = this.getBBox();
+            right = other.getBBox();
         } else {
-            return 0;
+            left = other.getBBox();
+            right = this.getBBox();
         }
+
+        return Math.max(0, right.getMinX() - left.getMaxX());
     }
 
 
     public double verticalDistance(BoundingBox other) {
 
-        double rect1Top = getMaxY();
-        double rect1Bottom = getMinY();
-        double rect2Top = other.getMaxY();
-        double rect2Bottom = other.getMinY();
-
-        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
-            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        Rectangle2D bottom;
+        Rectangle2D top;
+        if (this.isAbove(other)) {
+            top = this.getBBox();
+            bottom = other.getBBox();
         } else {
-            return 0;
+            bottom = this.getBBox();
+            top = other.getBBox();
         }
+
+        return Math.max(0, bottom.getMinY() - top.getMaxY());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -1,9 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD_ITALIC;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.ITALIC;
-import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.STANDARD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.BOLD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.BOLD_ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.STANDARD;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -14,7 +14,7 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;
 
 import lombok.Data;
@@ -24,7 +24,7 @@ import lombok.EqualsAndHashCode;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
 public class Line extends TextBoundingBox {
 
-    private static final double WORD_DISTANCE_MULTIPLIER = 0.17;
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.18;
 
     @EqualsAndHashCode.Include
     private final double x0;
@@ -41,7 +41,7 @@ public class Line extends TextBoundingBox {
     private FontStyle fontStyle;
 
     private final List<Character> characters;
-    private final List<Word> words = new ArrayList<>();
+    private final List<TextPositionSequence> words = new ArrayList<>();
 
 
     public Line(List<Character> characters, double wordSpacing) {
@@ -89,7 +89,7 @@ public class Line extends TextBoundingBox {
         for (FontStyle fontStyle : FontStyle.values()) {
             fontStyleCounter.put(fontStyle, new AtomicInteger(0));
         }
-        for (Word word : words) {
+        for (TextPositionSequence word : words) {
             switch (word.getFontStyle()) {
                 case STANDARD -> fontStyleCounter.get(FontStyle.REGULAR).getAndIncrement();
                 case BOLD -> fontStyleCounter.get(FontStyle.BOLD).getAndIncrement();
@@ -159,17 +159,14 @@ public class Line extends TextBoundingBox {
 
     private void computeWords(double wordSpacing) {
 
-        // Imo, the width of space should be scaled with the font size, but it only depends on the median distance between horizontal neighbours.
-        // If there are large differences in fontsize on a page, this might lead to missing spaces for the smaller fonts and too many for larger fonts.
-        // I've just now changed the scaling factor. If you come across this comment with missing whitespaces again, try scaling the fontsize instead of simply changing the factor again.
-        Word word = new Word();
+        TextPositionSequence word = new TextPositionSequence();
         Character previous = null;
         for (Character current : characters) {
             if (previous != null) {
                 double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
                 if (dist > wordSpacing) {
                     words.add(word);
-                    word = new Word();
+                    word = new TextPositionSequence();
                 }
             }
             word.getTextPositions().add(current.getTextPosition());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/TextBoundingBox.java

@@ -99,82 +99,4 @@ public abstract class TextBoundingBox extends BoundingBox {
         return this.bBoxDirAdj.getCenterX();
     }
 
-
-    public double horizontalDistanceDirAdj(TextBoundingBox other) {
-
-        double rect1Right = getMaxXDirAdj();
-        double rect1Left = getXDirAdj();
-        double rect2Right = other.getMaxXDirAdj();
-        double rect2Left = other.getXDirAdj();
-
-        if (rect1Left > rect2Right || rect2Left > rect1Right) {
-            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
-        } else {
-            return 0;
-        }
-    }
-
-
-    public double verticalDistanceDirAdj(TextBoundingBox other) {
-
-        double rect1Top = getMaxYDirAdj();
-        double rect1Bottom = getYDirAdj();
-        double rect2Top = other.getMaxYDirAdj();
-        double rect2Bottom = other.getYDirAdj();
-
-        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
-            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
-        } else {
-            return 0;
-        }
-    }
-
-
-    public boolean intersectsDirAdj(TextBoundingBox other) {
-
-        return this.intersectsXDirAdj(other) && this.intersectsYDirAdj(other);
-    }
-
-
-    public boolean intersectsDirAdj(TextBoundingBox other, float yThreshold, float xThreshold) {
-
-        return this.intersectsXDirAdj(other, xThreshold) && this.intersectsYDirAdj(other, yThreshold);
-    }
-
-
-    public boolean intersectsXDirAdj(TextBoundingBox other, float threshold) {
-
-        return this.getXDirAdj() - threshold <= other.getMaxXDirAdj() && this.getMaxXDirAdj() + threshold >= other.getXDirAdj();
-    }
-
-
-    public boolean intersectsXDirAdj(TextBoundingBox other) {
-
-        return this.getXDirAdj() <= other.getMaxXDirAdj() && this.getMaxXDirAdj() >= other.getXDirAdj();
-    }
-
-
-    public boolean intersectsYDirAdj(TextBoundingBox other) {
-
-        return this.getYDirAdj() <= other.getMaxYDirAdj() && this.getMaxYDirAdj() >= other.getYDirAdj();
-    }
-
-
-    public boolean intersectsYDirAdj(TextBoundingBox other, float threshold) {
-
-        return this.getYDirAdj() - threshold <= other.getMaxYDirAdj() && this.getMaxYDirAdj() + threshold >= other.getYDirAdj();
-    }
-
-
-    public boolean isAboveDirAdj(TextBoundingBox other) {
-
-        return other.isBelow(this);
-    }
-
-
-    public boolean isBelowDirAdj(TextBoundingBox other) {
-
-        return this.intersectsXDirAdj(other) && this.getYDirAdj() >= other.getMaxYDirAdj();
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/UnionFind.java

@@ -28,10 +28,4 @@ public class UnionFind<T> extends org.jgrapht.alg.util.UnionFind<T> {
         return setRep.values();
     }
 
-
-    public Collection<T> getElements() {
-
-        return getParentMap().keySet();
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/markdown/MarkdownMapper.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
+package com.knecon.fforesight.service.layoutparser.processor.markdown;
 
 import java.util.ArrayList;
 import java.util.HashSet;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/FloatFrequencyCounter.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -12,14 +13,10 @@ import lombok.Getter;
 public class FloatFrequencyCounter {
 
     Map<Double, Integer> countPerValue = new HashMap<>();
-    boolean changed;
-    Double mostPopularCache;
 
 
     public void add(double value) {
 
-        changed = true;
-
         if (!countPerValue.containsKey(value)) {
             countPerValue.put(value, 1);
         } else {
@@ -30,8 +27,6 @@ public class FloatFrequencyCounter {
 
     public void addAll(Map<Double, Integer> otherCounter) {
 
-        changed = true;
-
         for (Map.Entry<Double, Integer> entry : otherCounter.entrySet()) {
             if (countPerValue.containsKey(entry.getKey())) {
                 countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
@@ -44,27 +39,27 @@ public class FloatFrequencyCounter {
 
     public Double getMostPopular() {
 
-        if (changed || mostPopularCache == null) {
-            Map.Entry<Double, Integer> mostPopular = null;
-            for (Map.Entry<Double, Integer> entry : countPerValue.entrySet()) {
-                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
-                    mostPopular = entry;
-                }
+        Map.Entry<Double, Integer> mostPopular = null;
+        for (Map.Entry<Double, Integer> entry : countPerValue.entrySet()) {
+            if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
+                mostPopular = entry;
             }
-            mostPopularCache = mostPopular != null ? mostPopular.getKey() : 0;
-            changed = false;
         }
-
-        return mostPopularCache;
+        return mostPopular != null ? mostPopular.getKey() : null;
     }
 
 
-    public List<Double> getValuesInReverseOrder() {
+    public List<Double> getHigherThanMostPopular() {
 
-        return countPerValue.keySet()
-                .stream()
-                .sorted(Collections.reverseOrder())
-                .collect(Collectors.toList());
+        Double mostPopular = getMostPopular();
+        List<Double> higher = new ArrayList<>();
+        for (Double value : countPerValue.keySet()) {
+            if (value > mostPopular) {
+                higher.add(value);
+            }
+        }
+
+        return higher.stream().sorted(Collections.reverseOrder()).collect(Collectors.toList());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/LineInformation.java

@@ -3,7 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,8 +16,8 @@ import lombok.experimental.FieldDefaults;
 public class LineInformation {
 
     List<Rectangle2D> lineBBox;
-    List<List<Word>> sequencesByLines;
+    List<List<TextPositionSequence>> sequencesByLines;
     List<List<Rectangle2D>> bBoxWithGapsByLines;
-    List<List<List<Word>>> sequencesWithGapsByLines;
+    List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageBlockType.java

@@ -9,13 +9,12 @@ public enum PageBlockType {
     H6,
     HEADER,
     FOOTER,
+    TITLE,
     PARAGRAPH,
     PARAGRAPH_BOLD,
     PARAGRAPH_ITALIC,
     PARAGRAPH_UNKNOWN,
     OTHER,
-    TABLE_OF_CONTENTS_ITEM,
-    LIST_ITEM,
     TABLE;
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageContents.java

@@ -4,7 +4,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -15,7 +15,7 @@ import lombok.Getter;
 @AllArgsConstructor
 public class PageContents {
 
-    List<Word> sortedWords;
+    List<TextPositionSequence> sortedTextPositionSequences;
     Rectangle2D cropBox;
     Rectangle2D mediaBox;
     List<Ruling> rulings;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/SectionIdentifier.java

@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -16,13 +15,11 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {
 
-    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d{1,2})(?:[\\s.,;](\\d{1,2}))?(?:[\\s.,;](\\d{1,2}))?(?:[\\s.,;](\\d{1,2}))?");
-    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d{1,2})[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
 
     public enum Format {
         EMPTY,
         NUMERICAL,
-        ALPHANUMERIC,
         DOCUMENT
     }
 
@@ -44,10 +41,6 @@ public class SectionIdentifier {
         if (numericalIdentifierMatcher.find()) {
             return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher);
         }
-        Matcher alphanumericIdentifierMatcher = alphanumericIdentifierPattern.matcher(headline);
-        if (alphanumericIdentifierMatcher.find()) {
-            return buildAlphanumericSectionIdentifier(headline, alphanumericIdentifierMatcher);
-        }
         // more formats here
         return SectionIdentifier.empty();
     }
@@ -82,36 +75,7 @@ public class SectionIdentifier {
             }
             identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
         }
-        return new SectionIdentifier(Format.NUMERICAL,
-                                     identifierString,
-                                     identifiers.stream()
-                                             .toList(),
-                                     false);
-    }
-
-
-    private static SectionIdentifier buildAlphanumericSectionIdentifier(String headline, Matcher alphanumericIdentifierMatcher) {
-
-        String identifierString = headline.substring(alphanumericIdentifierMatcher.start(), alphanumericIdentifierMatcher.end());
-
-        String alphanumericIdentifier = alphanumericIdentifierMatcher.group(0).substring(0, 1).toUpperCase(Locale.ENGLISH);
-        int mappedCharacterValue = alphanumericIdentifier.charAt(0) - 'A' + 1;
-        List<Integer> identifiers = new LinkedList<>();
-        identifiers.add(mappedCharacterValue);
-
-        for (int i = 1; i <= 3; i++) {
-            String numericalIdentifier = alphanumericIdentifierMatcher.group(i);
-            if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) {
-                break;
-            }
-            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
-        }
-
-        return new SectionIdentifier(Format.ALPHANUMERIC,
-                                     identifierString,
-                                     identifiers.stream()
-                                             .toList(),
-                                     false);
+        return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false);
     }
 
 
@@ -159,22 +123,4 @@ public class SectionIdentifier {
         return identifierString;
     }
 
-
-    public boolean isEmpty() {
-
-        return this.format.equals(Format.EMPTY);
-    }
-
-
-    public int level() {
-
-        return identifiers.size();
-    }
-
-
-    protected List<Integer> getIdentifiers() {
-
-        return identifiers;
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/AbstractNodeVisitor.java

@@ -84,7 +84,7 @@ public abstract class AbstractNodeVisitor implements NodeVisitor {
     }
 
 
-    protected void visitChildren(SemanticNode semanticNode) {
+    private void visitChildren(SemanticNode semanticNode) {
 
         semanticNode.streamChildren()
                 .forEach(node -> node.accept(this));

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/TextRange.java

@@ -98,10 +98,10 @@ public class TextRange implements Comparable<TextRange> {
     public List<TextRange> split(List<Integer> splitIndices) {
 
         if (splitIndices.stream()
-                .anyMatch(idx -> !this.contains(idx))) {
+                .anyMatch(idx -> !this.containsExclusive(idx))) {
             throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s",
                                                        splitIndices.stream()
-                                                               .filter(idx -> !this.contains(idx))
+                                                               .filter(idx -> !this.containsExclusive(idx))
                                                                .toList(),
                                                        this));
         }
@@ -116,9 +116,7 @@ public class TextRange implements Comparable<TextRange> {
             splitBoundaries.add(new TextRange(previousIndex, splitIndex));
             previousIndex = splitIndex;
         }
-        if (previousIndex != end) {
-            splitBoundaries.add(new TextRange(previousIndex, end));
-        }
+        splitBoundaries.add(new TextRange(previousIndex, end));
         return splitBoundaries;
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/DuplicatedParagraph.java

@@ -25,4 +25,11 @@ public class DuplicatedParagraph extends Paragraph {
 
     }
 
+
+    @Override
+    public String toString() {
+
+        return super.toString();
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Page.java

@@ -68,7 +68,6 @@ public class Page {
     public TextBlock getMainBodyTextBlock() {
 
         return textBlocksOnPage.stream()
-                .filter(atb -> !atb.isEmpty())
                 .collect(new TextBlockCollector());
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java

@@ -22,10 +22,11 @@ public class ClassifiedImage {
     private boolean isAppendedToSection;
     private boolean hasTransparency;
     private int page;
+    @NonNull
     private String representation;
 
 
-    public ClassifiedImage(@NonNull Rectangle2D position, @NonNull ImageType imageType, boolean hasTransparency, int page, String representation) {
+    public ClassifiedImage(@NonNull Rectangle2D position, @NonNull ImageType imageType, boolean hasTransparency, int page, @NonNull String representation) {
 
         this.position = position;
         this.imageType = imageType;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineValidationService.java

@@ -10,7 +10,6 @@ import java.util.TreeSet;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import io.micrometer.observation.annotation.Observed;
@@ -21,9 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 public class OutlineValidationService {
 
     @Observed(name = "OutlineValidationService", contextualName = "create-toc")
-    public TableOfContents createToC(ClassificationDocument classificationDocument) {
-
-        List<TextPageBlock> headlines = extractHeadlines(classificationDocument);
+    public TableOfContents createToC(List<TextPageBlock> headlines) {
 
         List<TableOfContentItem> mainSections = new ArrayList<>();
         Map<Integer, TableOfContentItem> lastItemsPerDepth = new HashMap<>();
@@ -63,16 +60,4 @@ public class OutlineValidationService {
         return new TableOfContents(mainSections);
     }
 
-
-    private static List<TextPageBlock> extractHeadlines(ClassificationDocument classificationDocument) {
-
-        return classificationDocument.getPages()
-                .stream()
-                .flatMap(classificationPage -> classificationPage.getTextBlocks()
-                        .stream()
-                        .filter(tb -> tb instanceof TextPageBlock && tb.getClassification() != null && tb.getClassification().isHeadline())
-                        .map(tb -> (TextPageBlock) tb))
-                .toList();
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/TOCEnrichmentService.java

@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -186,8 +185,12 @@ public class TOCEnrichmentService {
             List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
             List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
             // Allow merging of tables if header row is separated from first logical non-header row
-            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows().get(0).size() == tableNonHeaderRow.size()) {
-                previousTableNonHeaderRow = previousTable.getRows().get(0)
+            if (previousTableNonHeaderRow.isEmpty()
+                && previousTable.getRowCount() == 1
+                && previousTable.getRows()
+                           .get(0).size() == tableNonHeaderRow.size()) {
+                previousTableNonHeaderRow = previousTable.getRows()
+                        .get(0)
                         .stream()
                         .map(cell -> {
                             Cell fakeCell = Cell.copy(cell);
@@ -198,7 +201,8 @@ public class TOCEnrichmentService {
             }
             if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
                 for (int i = currentTable.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-                    List<Cell> row = currentTable.getRows().get(i);
+                    List<Cell> row = currentTable.getRows()
+                            .get(i);
                     if (row.size() == tableNonHeaderRow.size() && row.stream()
                             .allMatch(cell -> cell.getHeaderCells().isEmpty())) {
                         for (int j = 0; j < row.size(); j++) {
@@ -221,15 +225,18 @@ public class TOCEnrichmentService {
 
         return table.getRows()
                 .stream()
-                .flatMap(Collection::stream)
-                .allMatch(cell -> cell.getHeaderCells().isEmpty());
+                .flatMap(row -> row.stream()
+                        .filter(cell -> !cell.getHeaderCells().isEmpty()))
+                .findAny().isEmpty();
+
     }
 
 
     private List<Cell> getRowWithNonHeaderCells(TablePageBlock table) {
 
         for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-            List<Cell> row = table.getRows().get(i);
+            List<Cell> row = table.getRows()
+                    .get(i);
             if (row.size() == 1) {
                 continue;
             }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/TableOfContentItem.java

@@ -5,7 +5,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
@@ -24,7 +24,7 @@ public class TableOfContentItem {
     private List<AbstractPageBlock> sectionBlocks = new ArrayList<>();
     private List<ClassifiedImage> images = new ArrayList<>();
 
-    private GenericSemanticNode section;
+    private AbstractSemanticNode section;
 
 
     public TableOfContentItem(TextPageBlock headline) {
@@ -45,7 +45,8 @@ public class TableOfContentItem {
         if (parent != null) {
             int index = parent.getChildren().indexOf(this);
             if (index > 0) {
-                return parent.getChildren().get(index - 1);
+                return parent.getChildren()
+                        .get(index - 1);
             }
         }
         return null;
@@ -57,7 +58,8 @@ public class TableOfContentItem {
         if (parent != null) {
             int index = parent.getChildren().indexOf(this);
             if (index >= 0 && index < parent.getChildren().size() - 1) {
-                return parent.getChildren().get(index + 1);
+                return parent.getChildren()
+                        .get(index + 1);
             }
         }
         return null;
@@ -91,19 +93,17 @@ public class TableOfContentItem {
         return false;
     }
 
-
     public List<AbstractPageBlock> getNonEmptySectionBlocks() {
 
-        return sectionBlocks.stream()
-                .filter(pageBlock -> !pageBlock.isEmpty())
-                .collect(Collectors.toList());
+        return sectionBlocks.stream().filter(pageBlock -> !pageBlock.isEmpty()).collect(Collectors.toList());
     }
 
-
     @Override
     public String toString() {
 
         return "OutlineObjectTreeNode{" + "textPageBlock=" + headline + '}';
     }
 
+
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Cell.java

@@ -9,7 +9,7 @@ import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Data;
@@ -68,12 +68,12 @@ public class Cell extends BoundingBox {
         StringBuilder sb = new StringBuilder();
 
         Iterator<TextPageBlock> itty = textBlocks.iterator();
-        Word previous = null;
+        TextPositionSequence previous = null;
         while (itty.hasNext()) {
 
             TextPageBlock textBlock = itty.next();
 
-            for (Word word : textBlock.getWords()) {
+            for (TextPositionSequence word : textBlock.getSequences()) {
                 if (previous != null) {
                     if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
                         sb.append('\n');

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/AbstractBlockOnPage.java

@@ -1,8 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-
-public record AbstractBlockOnPage(AbstractPageBlock block, ClassificationPage page) {
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/FrequencyCounters.java

@@ -1,21 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@NoArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class FrequencyCounters {
-
-    FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
-    FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
-    FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
-    StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
-    StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/ListIdentifier.java

@@ -1,107 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ListIdentifier {
-
-    public static final Pattern NUMBER_WITH_DOT = Pattern.compile("^\\s*([1-9]{1,4})\\.\\s+");
-    public static final Pattern NUMBER_IN_PARENTHESES = Pattern.compile("^\\s*\\(([1-9]{1,4})\\)\\s+");
-
-    enum Format {
-        NUMBER_WITH_DOT,
-        NUMBER_IN_PARENTHESES
-    }
-
-    Format format;
-    @Getter
-    Word word;
-    @Getter
-    int page;
-    int representation;
-
-
-    public static Optional<ListIdentifier> parse(TextPageBlock textPageBlock, int page) {
-
-        return parse(textPageBlock.getWords().subList(0, Math.min(5, textPageBlock.getWords().size())), page);
-    }
-
-
-    public static Optional<ListIdentifier> parse(List<Word> sequences, int page) {
-
-        StringBuilder sb = new StringBuilder();
-        for (Word sequence : sequences) {
-            sb.append(sequence.toString());
-            sb.append(" ");
-        }
-        sb.replace(sb.length() - 1, sb.length(), "");
-        String text = sb.toString();
-
-        Matcher numberMatcher = NUMBER_WITH_DOT.matcher(text);
-
-        if (numberMatcher.find()) {
-            Optional<Integer> representation = parseInteger(numberMatcher.group(1));
-            if (representation.isPresent()) {
-                return Optional.of(new ListIdentifier(Format.NUMBER_WITH_DOT, sequences.get(0), page, representation.get()));
-            }
-        }
-
-        Matcher parenthesisMatcher = NUMBER_IN_PARENTHESES.matcher(text);
-        if (parenthesisMatcher.find()) {
-            Optional<Integer> representation = parseInteger(parenthesisMatcher.group(1));
-            if (representation.isPresent()) {
-                return Optional.of(new ListIdentifier(Format.NUMBER_IN_PARENTHESES, sequences.get(0), page, representation.get()));
-            }
-        }
-        return Optional.empty();
-    }
-
-
-    private static Optional<Integer> parseInteger(String text) {
-
-        try {
-            return Optional.of(Integer.parseInt(text));
-        } catch (NumberFormatException e) {
-            return Optional.empty();
-        }
-    }
-
-
-    public static boolean isInOrder(List<ListIdentifier> listIdentifiers) {
-
-        if (listIdentifiers.size() <= 1) {
-            return true;
-        }
-
-        for (int i = 1; i < listIdentifiers.size(); i++) {
-            ListIdentifier current = listIdentifiers.get(i);
-            ListIdentifier previous = listIdentifiers.get(i - 1);
-            if (current.format != previous.format) {
-                return false;
-            }
-            if (current.representation <= previous.representation) {
-                return false;
-            }
-            if (!current.word.intersectsXDirAdj(previous.word, 2)) {
-                return false;
-            }
-            if (current.page == previous.page && !current.word.isBelowDirAdj(previous.word)) {
-                return false;
-            }
-            if (current.page < previous.page) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -7,8 +7,6 @@ import org.apache.pdfbox.text.TextPosition;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
-import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/SearchableText.java

@@ -10,18 +10,18 @@ import lombok.Getter;
 @Getter
 public class SearchableText {
 
-    private final List<Word> sequences = new ArrayList<>();
+    private final List<TextPositionSequence> sequences = new ArrayList<>();
 
 
-    public void add(Word word) {
+    public void add(TextPositionSequence textPositionSequence) {
 
-        sequences.add(word);
+        sequences.add(textPositionSequence);
     }
 
 
-    public void addAll(List<Word> words) {
+    public void addAll(List<TextPositionSequence> textPositionSequences) {
 
-        sequences.addAll(words);
+        sequences.addAll(textPositionSequences);
     }
 
 
@@ -32,10 +32,10 @@ public class SearchableText {
     }
 
 
-    public static String buildString(List<Word> sequences) {
+    public static String buildString(List<TextPositionSequence> sequences) {
 
         StringBuilder sb = new StringBuilder();
-        for (Word word : sequences) {
+        for (TextPositionSequence word : sequences) {
             sb.append(word);
             sb.append(' ');
         }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/StringFrequencyCounter.java

@@ -9,14 +9,10 @@ public class StringFrequencyCounter {
 
     @Getter
     private final Map<String, Integer> countPerValue = new HashMap<>();
-    boolean changed;
-    String mostPopularCache;
 
 
     public void add(String value) {
 
-        changed = true;
-
         if (!countPerValue.containsKey(value)) {
             countPerValue.put(value, 1);
         } else {
@@ -27,8 +23,6 @@ public class StringFrequencyCounter {
 
     public void addAll(Map<String, Integer> otherCounter) {
 
-        changed = true;
-
         for (Map.Entry<String, Integer> entry : otherCounter.entrySet()) {
             if (countPerValue.containsKey(entry.getKey())) {
                 countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
@@ -41,18 +35,13 @@ public class StringFrequencyCounter {
 
     public String getMostPopular() {
 
-        if (changed || mostPopularCache == null) {
-            Map.Entry<String, Integer> mostPopular = null;
-            for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
-                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
-                    mostPopular = entry;
-                }
+        Map.Entry<String, Integer> mostPopular = null;
+        for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
+            if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
+                mostPopular = entry;
             }
-            mostPopularCache = mostPopular != null ? mostPopular.getKey() : null;
-            changed = false;
         }
-
-        return mostPopularCache;
+        return mostPopular != null ? mostPopular.getKey() : null;
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextBlockOnPage.java

@@ -1,7 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-
-public record TextBlockOnPage(TextPageBlock textBlock, ClassificationPage page) {
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java

@@ -2,11 +2,11 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
@@ -25,59 +25,56 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {
 
     @Builder.Default
-    private List<Word> words = new ArrayList<>();
-    @Builder.Default
-    private FrequencyCounters frequencyCounters = new FrequencyCounters();
+    private List<TextPositionSequence> sequences = new ArrayList<>();
 
     private Rectangle2D bBoxDirAdj;
 
-    private boolean underlined;
+    private String mostPopularWordFont;
+
+    private String mostPopularWordStyle;
+
+    private double mostPopularWordFontSize;
+
+    private double mostPopularWordHeight;
+
+    private double mostPopularWordSpaceWidth;
+
+    private double highestFontSize;
 
     private PageBlockType classification;
 
     private boolean toDuplicate;
 
-    private String text;
-    private boolean changed;
 
+    public TextPageBlock(List<TextPositionSequence> sequences) {
 
-    public TextPageBlock(List<Word> words) {
-
-        this.words = new ArrayList<>(words);
-        this.frequencyCounters = new FrequencyCounters();
-
-        if (!words.isEmpty()) {
-            addToFrequencyCounters(words);
+        this.sequences = sequences;
+        if (!sequences.isEmpty()) {
+            calculateFrequencyCounters();
         }
         calculateBBox();
     }
 
 
-    public List<Word> getWords() {
-
-        return Collections.unmodifiableList(words);
-    }
-
-
     public TextDirection getDir() {
 
-        return words.get(0).getDir();
+        return sequences.get(0).getDir();
     }
 
 
     private void calculateBBox() {
 
-        if (words == null) {
+        if (sequences == null) {
             this.bBox = new Rectangle2D.Double();
             this.bBoxPdf = new Rectangle2D.Double();
             this.bBoxDirAdj = new Rectangle2D.Double();
             return;
         }
-        this.bBoxDirAdj = words.stream()
-                .map(Word::getBBoxDirAdj)
+        this.bBoxDirAdj = sequences.stream()
+                .map(TextPositionSequence::getBBoxDirAdj)
                 .collect(RectangleTransformations.collectBBox());
 
-        setToBBoxOfComponents(words);
+        setToBBoxOfComponents(sequences);
     }
 
 
@@ -99,8 +96,8 @@ public class TextPageBlock extends AbstractPageBlock {
             throw new IllegalArgumentException("Cannot merge textBlocks on different pages.");
         }
 
-        List<Word> sequences = textBlocksToMerge.stream()
-                .map(TextPageBlock::getWords)
+        List<TextPositionSequence> sequences = textBlocksToMerge.stream()
+                .map(TextPageBlock::getSequences)
                 .flatMap(java.util.Collection::stream)
                 .toList();
         sequences = new ArrayList<>(sequences);
@@ -109,27 +106,38 @@ public class TextPageBlock extends AbstractPageBlock {
     }
 
 
-    private void addToFrequencyCounters(List<Word> sequences) {
+    private void calculateFrequencyCounters() {
 
-        for (Word wordBlock : sequences) {
+        FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
+        FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
+        FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
+        StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
+        StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
+
+        for (TextPositionSequence wordBlock : sequences) {
+
+            lineHeightFrequencyCounter.add(wordBlock.getTextHeight());
+            fontSizeFrequencyCounter.add(wordBlock.getFontSize());
+            spaceFrequencyCounter.add(wordBlock.getSpaceWidth());
+            fontFrequencyCounter.add(wordBlock.getFont());
+            styleFrequencyCounter.add(wordBlock.getFontStyle());
 
-            frequencyCounters.getLineHeightFrequencyCounter().add(wordBlock.getTextHeight());
-            frequencyCounters.getFontSizeFrequencyCounter().add(wordBlock.getFontSize());
-            frequencyCounters.getSpaceFrequencyCounter().add(wordBlock.getSpaceWidth());
-            frequencyCounters.getFontFrequencyCounter().add(wordBlock.getFont());
-            frequencyCounters.getStyleFrequencyCounter().add(wordBlock.getFontStyle());
         }
 
-        setUnderlined(this.words.stream()
-                              .allMatch(Word::isUnderline));
+        setMostPopularWordFont(fontFrequencyCounter.getMostPopular());
+        setMostPopularWordStyle(styleFrequencyCounter.getMostPopular());
+        setMostPopularWordFontSize(fontSizeFrequencyCounter.getMostPopular());
+        setMostPopularWordHeight(lineHeightFrequencyCounter.getMostPopular());
+        setMostPopularWordSpaceWidth(spaceFrequencyCounter.getMostPopular());
+        setHighestFontSize(fontSizeFrequencyCounter.getHighest());
     }
 
 
-    public TextPageBlock union(Word r) {
+    public TextPageBlock union(TextPositionSequence r) {
 
         TextPageBlock union = this.copy();
-        union.add(r);
-        addToFrequencyCounters(List.of(r));
+        union.getSequences().add(r);
+        calculateFrequencyCounters();
         calculateBBox();
         return union;
     }
@@ -138,50 +146,51 @@ public class TextPageBlock extends AbstractPageBlock {
     public TextPageBlock union(TextPageBlock r) {
 
         TextPageBlock union = this.copy();
-        union.addAll(r.getWords());
-        addToFrequencyCounters(r.getWords());
+        union.getSequences().addAll(r.getSequences());
+        calculateFrequencyCounters();
         calculateBBox();
         return union;
     }
 
 
-    public void add(TextPageBlock textPageBlock) {
+    public void add(TextPageBlock r) {
 
-        changed = true;
-        words.addAll(textPageBlock.getWords());
-        addToFrequencyCounters(textPageBlock.getWords());
+        sequences.addAll(r.getSequences());
+        calculateFrequencyCounters();
         calculateBBox();
     }
 
 
-    public void add(Word word) {
+    public void add(TextPositionSequence r) {
 
-        changed = true;
-        words.add(word);
-        addToFrequencyCounters(List.of(word));
-        calculateBBox();
-    }
-
-
-    public void addAll(List<Word> words) {
-
-        changed = true;
-        this.words.addAll(words);
-        addToFrequencyCounters(words);
+        sequences.add(r);
+        calculateFrequencyCounters();
         calculateBBox();
     }
 
 
     public TextPageBlock copy() {
 
-        return new TextPageBlock(new ArrayList<>(words));
+        return new TextPageBlock(new ArrayList<>(sequences));
     }
 
 
     @Override
     public String toString() {
 
-        return getText();
+        StringBuilder builder = new StringBuilder();
+
+        for (int i = 0; i < sequences.size(); i++) {
+            String sequenceAsString = sequences.get(i).toString();
+            // Fix for missing Whitespace. This is recognized in getSequences method. See PDFTextStripper Line 1730.
+            if (i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() - 1) != ' ' && sequenceAsString.charAt(0) != ' ') {
+                builder.append(' ');
+            }
+            builder.append(sequenceAsString);
+        }
+
+        return builder.toString();
+
     }
 
 
@@ -189,36 +198,30 @@ public class TextPageBlock extends AbstractPageBlock {
     @JsonIgnore
     public String getText() {
 
-        if (text == null || changed) {
+        StringBuilder sb = new StringBuilder();
 
-            StringBuilder sb = new StringBuilder();
-
-            Word previous = null;
-            for (Word word : words) {
-                if (previous != null) {
-                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
-                        sb.append('\n');
-                    } else {
-                        sb.append(' ');
-                    }
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : sequences) {
+            if (previous != null) {
+                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
+                    sb.append('\n');
+                } else {
+                    sb.append(' ');
                 }
-                sb.append(word.toString());
-                previous = word;
             }
-
-            text = TextNormalizationUtilities.removeHyphenLinebreaks(sb.toString());
-            changed = false;
+            sb.append(word.toString());
+            previous = word;
         }
 
-        return text;
+        return TextNormalizationUtilities.removeHyphenLinebreaks(sb.toString());
     }
 
 
     public int getNumberOfLines() {
 
         int numberOfLines = 1;
-        Word previous = null;
-        for (Word word : words) {
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : sequences) {
             if (previous != null) {
                 if (word.getMaxYDirAdj() - previous.getMaxYDirAdj() > word.getTextHeight()) {
                     numberOfLines++;
@@ -230,47 +233,10 @@ public class TextPageBlock extends AbstractPageBlock {
     }
 
 
-    public String getMostPopularWordFont() {
-
-        return frequencyCounters.getFontFrequencyCounter().getMostPopular();
-    }
-
-
-    public String getMostPopularWordStyle() {
-
-        return frequencyCounters.getStyleFrequencyCounter().getMostPopular();
-    }
-
-
-    public double getMostPopularWordFontSize() {
-
-        return frequencyCounters.getFontSizeFrequencyCounter().getMostPopular();
-    }
-
-
-    public double getMostPopularWordHeight() {
-
-        return frequencyCounters.getLineHeightFrequencyCounter().getMostPopular();
-    }
-
-
-    public double getMostPopularWordSpaceWidth() {
-
-        return frequencyCounters.getSpaceFrequencyCounter().getMostPopular();
-    }
-
-
-    public double getHighestFontSize() {
-
-        Double highest = frequencyCounters.getFontSizeFrequencyCounter().getHighest();
-        return highest == null ? 0 : highest;
-    }
-
-
     @Override
     public boolean isEmpty() {
 
-        return words.isEmpty();
+        return sequences.isEmpty();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -2,13 +2,9 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import static com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition.HEIGHT_PADDING;
 
-import java.awt.geom.AffineTransform;
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
-import java.util.Objects;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.apache.pdfbox.text.TextPosition;
@@ -18,6 +14,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextB
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -26,17 +23,18 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-@SuppressWarnings("pmd")
-public class Word extends TextBoundingBox implements CharSequence {
+@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true) // needs the bbox to be unique
+public class TextPositionSequence extends TextBoundingBox implements CharSequence {
 
     public static final String STANDARD = "standard";
     public static final String BOLD_ITALIC = "bold, italic";
     public static final String BOLD = "bold";
     public static final String ITALIC = "italic";
-    public static final Pattern FONT_CLEANER = Pattern.compile(",bold|,italic");
 
+    @EqualsAndHashCode.Include
     private int page;
 
+    @EqualsAndHashCode.Include
     @Builder.Default
     private List<RedTextPosition> textPositions = new ArrayList<>();
 
@@ -44,32 +42,29 @@ public class Word extends TextBoundingBox implements CharSequence {
     private boolean strikethrough;
     private boolean underline;
 
-    private Integer hashcodeCache;
 
-
-    public Word(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
+    public TextPositionSequence(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
 
         this.textPositions = textPositions.stream()
                 .map(RedTextPosition::fromTextPosition)
                 .collect(Collectors.toList());
         this.page = pageNumber;
         this.isParagraphStart = isParagraphStart;
-        calculateBBoxAndHashcode();
+        calculateBBox();
     }
 
 
-    private void calculateBBoxAndHashcode() {
+    private void calculateBBox() {
 
         setToBBoxOfComponents(getTextPositions());
-        hashcodeCache = null;
     }
 
 
-    public Word(List<RedTextPosition> textPositions, int page) {
+    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
 
         this.textPositions = textPositions;
         this.page = page;
-        calculateBBoxAndHashcode();
+        calculateBBox();
     }
 
 
@@ -98,9 +93,9 @@ public class Word extends TextBoundingBox implements CharSequence {
 
 
     @Override
-    public Word subSequence(int start, int end) {
+    public TextPositionSequence subSequence(int start, int end) {
 
-        var textPositionSequence = new Word();
+        var textPositionSequence = new TextPositionSequence();
         textPositionSequence.textPositions = textPositions.subList(start, end);
         textPositionSequence.page = page;
         textPositionSequence.dir = dir;
@@ -126,21 +121,20 @@ public class Word extends TextBoundingBox implements CharSequence {
     }
 
 
-    public void add(Word word, RedTextPosition textPosition) {
+    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {
 
         this.textPositions.add(textPosition);
-        this.page = word.getPage();
-        calculateBBoxAndHashcode();
+        this.page = textPositionSequence.getPage();
+        calculateBBox();
     }
 
 
     public void add(TextPosition textPosition) {
 
         this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
-        calculateBBoxAndHashcode();
+        calculateBBox();
     }
 
-
     public double getTextHeightNoPadding() {
 
         return textPositions.get(0).getHeightDirAdj();
@@ -158,8 +152,7 @@ public class Word extends TextBoundingBox implements CharSequence {
         if (textPositions.get(0).getFontName() == null) {
             return "none";
         }
-
-        return FONT_CLEANER.matcher(textPositions.get(0).getFontName().toLowerCase(Locale.ROOT)).replaceAll("");
+        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
     }
 
 
@@ -193,65 +186,5 @@ public class Word extends TextBoundingBox implements CharSequence {
         return textPositions.get(0).getWidthOfSpace();
     }
 
-
-    public boolean equals(final Object o) {
-        // auto-generated with lombok
-        if (o == this) {
-            return true;
-        }
-        if (!(o instanceof Word other)) {
-            return false;
-        }
-        if (!other.canEqual((Object) this)) {
-            return false;
-        }
-        if (!super.equals(o)) {
-            return false;
-        }
-        if (this.getPage() != other.getPage()) {
-            return false;
-        }
-        final Object this$textPositions = this.getTextPositions();
-        final Object other$textPositions = other.getTextPositions();
-        if (!Objects.equals(this$textPositions, other$textPositions)) {
-            return false;
-        }
-        return Objects.equals(this.getHashcodeCache(), other.getHashcodeCache());
-    }
-
-
-    protected boolean canEqual(final Object other) {return other instanceof Word;}
-
-
-    public int hashCode() {
-
-        if (hashcodeCache == null) {
-            hashcodeCache = hashcodeCalculation();
-        }
-
-        return hashcodeCache;
-    }
-
-
-    private int hashcodeCalculation() {
-
-        final int PRIME = 59;
-        int result = super.hashCode();
-        result = result * PRIME + this.getPage();
-        final Object $textPositions = this.getTextPositions();
-        result = result * PRIME + ($textPositions == null ? 43 : $textPositions.hashCode());
-        return result;
-    }
-
-
-    public void transform(AffineTransform rotateInstance) {
-
-        for (RedTextPosition textPosition : getTextPositions()) {
-            Rectangle2D exactDirAdjCoordinates = rotateInstance.createTransformedShape(textPosition.getBBoxDirAdj()).getBounds2D();
-            textPosition.setBBoxDirAdj(exactDirAdjCoordinates);
-        }
-        calculateBBoxAndHashcode();
-    }
-
 }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TocNumberComparator.java

@@ -1,36 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
-
-import java.util.Comparator;
-import java.util.HashMap;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.NumberWord;
-
-public class TocNumberComparator implements Comparator<NumberWord> {
-
-    private HashMap<NumberWord, TextBlockOnPage> lookup;
-
-
-    public TocNumberComparator(HashMap<NumberWord, TextBlockOnPage> lookup) {
-
-        this.lookup = lookup;
-    }
-
-
-    @Override
-    public int compare(NumberWord number1, NumberWord number2) {
-
-        int page1 = lookup.get(number1).page().getPageNumber();
-        int page2 = lookup.get(number2).page().getPageNumber();
-
-        if (page1 != page2) {
-            return Integer.compare(page1, page2);
-        }
-
-        if (number1.word().getY() != number2.word().getY()) {
-            return Double.compare(number1.word().getY(), number2.word().getY());
-        }
-
-        return Integer.compare(number1.number(), number2.number());
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/DividingColumnDetectionService.java

@@ -23,11 +23,11 @@ public class DividingColumnDetectionService {
     public List<Rectangle2D> detectColumns(PageContents pageContents) {
 
 
-        if (pageContents.getSortedWords().size() < 2) {
+        if (pageContents.getSortedTextPositionSequences().size() < 2) {
             return List.of(pageContents.getCropBox());
         }
 
-        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), pageContents.getCropBox());
+        GapInformation linesWithGapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), pageContents.getCropBox());
 
         return detectColumnsFromLines(linesWithGapInformation.getXGaps(), pageContents.getCropBox());
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/GapDetectionService.java

@@ -5,7 +5,7 @@ import java.util.LinkedList;
 import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AllArgsConstructor;
 import lombok.experimental.UtilityClass;
@@ -18,23 +18,23 @@ public class GapDetectionService {
     private static final double NEW_LINE_FACTOR = 0.2;
 
 
-    public static GapInformation findGapsInLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {
+    public static GapInformation findGapsInLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
 
-        if (sortedWords.isEmpty()) {
+        if (sortedTextPositionSequences.isEmpty()) {
             return new GapInformation();
         }
 
-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
 
         XGapsContext xGapContext = XGapsContext.init(mainBodyTextFrame);
         YGapsContext yGapContext = YGapsContext.init(mainBodyTextFrame);
 
-        var previousTextPosition = sortedWords.get(0);
+        var previousTextPosition = sortedTextPositionSequences.get(0);
         Rectangle2D rectangle = toRectangle2D(previousTextPosition);
 
         xGapContext.addGapFromLeftEdgeOfMainBody(rectangle);
 
-        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {
+        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
 
             double yDifference = Math.abs(currentTextPosition.getMaxYDirAdj() - previousTextPosition.getMaxYDirAdj());
             double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj());
@@ -59,14 +59,14 @@ public class GapDetectionService {
             }
             previousTextPosition = currentTextPosition;
         }
-        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedWords.get(sortedWords.size() - 1)));
+        xGapContext.addGapToRightEdgeOfMainBody(toRectangle2D(sortedTextPositionSequences.get(sortedTextPositionSequences.size() - 1)));
         xGapContext.gapsPerLine.add(xGapContext.gapsInCurrentLine);
 
         return new GapInformation(xGapContext.gapsPerLine, yGapContext.gapsPerLine);
     }
 
 
-    private static Rectangle2D toRectangle2D(Word textPosition) {
+    private static Rectangle2D toRectangle2D(TextPositionSequence textPosition) {
 
         return mirrorY(textPosition.getBBox());
     }
@@ -87,18 +87,18 @@ public class GapDetectionService {
     }
 
 
-    private static void assertAllTextPositionsHaveSameDir(List<Word> words) {
+    private static void assertAllTextPositionsHaveSameDir(List<TextPositionSequence> textPositionSequences) {
 
-        assert words.stream()
-                .map(Word::getDir)
-                .allMatch(a -> a.equals(words.get(0).getDir()));
+        assert textPositionSequences.stream()
+                .map(TextPositionSequence::getDir)
+                .allMatch(a -> a.equals(textPositionSequences.get(0).getDir()));
     }
 
 
-    private static double getAvgTextPositionHeight(List<Word> words) {
+    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
 
-        return words.stream()
-                .mapToDouble(Word::getHeight).average().orElseThrow();
+        return textPositionSequences.stream()
+                .mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/InvisibleTableDetectionService.java

@@ -7,17 +7,17 @@ import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
 public class InvisibleTableDetectionService {
 
-    public List<List<Rectangle2D>> detectTable(List<Word> words, Rectangle2D tableBBox) {
+    public List<List<Rectangle2D>> detectTable(List<TextPositionSequence> textPositionSequences, Rectangle2D tableBBox) {
 
-        LineInformation lineInformation = LineDetectionService.calculateLineInformation(words);
-        GapInformation gaps = GapDetectionService.findGapsInLines(words, tableBBox);
+        LineInformation lineInformation = LineDetectionService.calculateLineInformation(textPositionSequences);
+        GapInformation gaps = GapDetectionService.findGapsInLines(textPositionSequences, tableBBox);
         List<Rectangle2D> gapsAcrossLines = GapsAcrossLinesService.detectXGapsAcrossLines(gaps, tableBBox);
         List<Double> columnXCoords = gapsAcrossLines.stream().map(RectangularShape::getCenterX).toList();
         int colCount = gapsAcrossLines.size();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/LineDetectionService.java

@@ -7,7 +7,7 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -19,37 +19,37 @@ public class LineDetectionService {
     private static final double X_GAP_FACTOR = 1; // multiplied with average text height, determines the minimum distance of gaps in lines
 
 
-    public LineInformation calculateLineInformation(List<Word> sortedWords) {
+    public LineInformation calculateLineInformation(List<TextPositionSequence> sortedTextPositionSequences) {
 
-        if (sortedWords.isEmpty()) {
+        if (sortedTextPositionSequences.isEmpty()) {
             return LineFactory.init().build();
         }
 
-        return buildLineInformation(sortedWords);
+        return buildLineInformation(sortedTextPositionSequences);
     }
 
 
-    public List<List<Rectangle2D>> findLinesWithGaps(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {
+    public List<List<Rectangle2D>> findLinesWithGaps(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
 
-        return calculateLineInformation(sortedWords).getBBoxWithGapsByLines();
+        return calculateLineInformation(sortedTextPositionSequences).getBBoxWithGapsByLines();
     }
 
 
-    public List<List<Word>> orderByLines(List<Word> sortedWords, Rectangle2D mainBodyTextFrame) {
+    public List<List<TextPositionSequence>> orderByLines(List<TextPositionSequence> sortedTextPositionSequences, Rectangle2D mainBodyTextFrame) {
 
-        return calculateLineInformation(sortedWords).getSequencesByLines();
+        return calculateLineInformation(sortedTextPositionSequences).getSequencesByLines();
     }
 
 
-    private static LineInformation buildLineInformation(List<Word> sortedWords) {
+    private static LineInformation buildLineInformation(List<TextPositionSequence> sortedTextPositionSequences) {
 
-        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedWords);
+        final double avgTextPositionHeight = getAvgTextPositionHeight(sortedTextPositionSequences);
 
         LineFactory lineFactory = LineFactory.init();
 
-        var previousTextPosition = sortedWords.get(0);
+        var previousTextPosition = sortedTextPositionSequences.get(0);
         lineFactory.addToCurrentLine(previousTextPosition);
-        for (Word currentTextPosition : sortedWords.subList(1, sortedWords.size())) {
+        for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
             if (isNewLine(currentTextPosition, previousTextPosition, avgTextPositionHeight) || isSplitByOrientation(currentTextPosition, previousTextPosition)) {
                 lineFactory.startNewLine();
             } else if (isXGap(currentTextPosition, previousTextPosition, avgTextPositionHeight)) {
@@ -63,25 +63,25 @@ public class LineDetectionService {
     }
 
 
-    private static double getAvgTextPositionHeight(List<Word> words) {
+    private static double getAvgTextPositionHeight(List<TextPositionSequence> textPositionSequences) {
 
-        return words.stream().mapToDouble(Word::getHeight).average().orElseThrow();
+        return textPositionSequences.stream().mapToDouble(TextPositionSequence::getHeight).average().orElseThrow();
     }
 
 
-    private static boolean isXGap(Word currentTextPosition, Word previousTextPosition, double avgTextPositionHeight) {
+    private static boolean isXGap(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
 
         return Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj()) > (avgTextPositionHeight * X_GAP_FACTOR);
     }
 
 
-    private static boolean isSplitByOrientation(Word currentTextPosition, Word previousTextPosition) {
+    private static boolean isSplitByOrientation(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition) {
 
         return !previousTextPosition.getDir().equals(currentTextPosition.getDir());
     }
 
 
-    private static boolean isNewLine(Word currentTextPosition, Word previousTextPosition, double avgTextPositionHeight) {
+    private static boolean isNewLine(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
 
         return Math.abs(previousTextPosition.getYDirAdj() - currentTextPosition.getYDirAdj()) > avgTextPositionHeight;
     }
@@ -96,13 +96,13 @@ public class LineDetectionService {
         List<List<Rectangle2D>> bBoxWithGapsByLines;
         List<Rectangle2D> bBoxWithGapsInCurrentLine;
 
-        List<List<List<Word>>> sequencesWithGapsByLines;
-        List<List<Word>> sequencesWithGapsInCurrentLine;
+        List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
+        List<List<TextPositionSequence>> sequencesWithGapsInCurrentLine;
 
-        List<Word> currentSequencesWithoutGaps;
+        List<TextPositionSequence> currentSequencesWithoutGaps;
 
-        List<List<Word>> sequencesByLines;
-        List<Word> sequencesInCurrentLine;
+        List<List<TextPositionSequence>> sequencesByLines;
+        List<TextPositionSequence> sequencesInCurrentLine;
 
         List<List<Rectangle2D>> xGaps;
         List<List<Rectangle2D>> yGaps;
@@ -116,14 +116,14 @@ public class LineDetectionService {
             List<Rectangle2D> bBoxWithGapsInCurrentLine = new LinkedList<>();
             bBoxWithGapsByLines.add(bBoxWithGapsInCurrentLine);
 
-            List<List<List<Word>>> sequencesWithGapsByLines = new LinkedList<>();
-            List<List<Word>> sequencesWithGapsInCurrentLine = new LinkedList<>();
+            List<List<List<TextPositionSequence>>> sequencesWithGapsByLines = new LinkedList<>();
+            List<List<TextPositionSequence>> sequencesWithGapsInCurrentLine = new LinkedList<>();
             sequencesWithGapsByLines.add(sequencesWithGapsInCurrentLine);
-            List<Word> currentSequencesWithoutGaps = new LinkedList<>();
+            List<TextPositionSequence> currentSequencesWithoutGaps = new LinkedList<>();
             sequencesWithGapsInCurrentLine.add(currentSequencesWithoutGaps);
 
-            List<List<Word>> sequencesByLines = new LinkedList<>();
-            List<Word> sequencesInCurrentLine = new LinkedList<>();
+            List<List<TextPositionSequence>> sequencesByLines = new LinkedList<>();
+            List<TextPositionSequence> sequencesInCurrentLine = new LinkedList<>();
             sequencesByLines.add(sequencesInCurrentLine);
 
             return new LineFactory(lineBBox,
@@ -178,13 +178,13 @@ public class LineDetectionService {
         }
 
 
-        private Rectangle2D textPositionBBox(List<Word> words) {
+        private Rectangle2D textPositionBBox(List<TextPositionSequence> textPositionSequences) {
 
-            return RectangleTransformations.rectangle2DBBox(words.stream().map(Word::getBBox).toList());
+            return RectangleTransformations.rectangle2DBBox(textPositionSequences.stream().map(TextPositionSequence::getBBox).toList());
         }
 
 
-        public void addToCurrentLine(Word current) {
+        public void addToCurrentLine(TextPositionSequence current) {
 
             sequencesInCurrentLine.add(current);
             currentSequencesWithoutGaps.add(current);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PageContentExtractor.java

@@ -13,7 +13,7 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.springframework.core.io.ClassPathResource;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
@@ -40,7 +40,7 @@ public class PageContentExtractor {
                 stripper.setPdpage(pdPage);
                 stripper.getText(pdDocument);
 
-                Map<Float, List<Word>> sortedTextPositionSequencesPerDir = stripper.getWords()
+                Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
                         .stream()
                         .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
 
@@ -57,7 +57,7 @@ public class PageContentExtractor {
     }
 
 
-    public List<Word> sortByDirAccordingToPageRotation(Map<Float, List<Word>> sortedTextPositionSequencesPerDir, int rotation) {
+    public List<TextPositionSequence> sortByDirAccordingToPageRotation(Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir, int rotation) {
 
         LinkedList<Float> sortedKeys = new LinkedList<>(sortedTextPositionSequencesPerDir.keySet().stream().sorted().toList());
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PageInformationService.java

@@ -14,9 +14,9 @@ public class PageInformationService {
 
     public PageInformation build(PageContents pageContents) {
 
-        LineInformation lineInformation = LineDetectionService.calculateLineInformation(pageContents.getSortedWords());
+        LineInformation lineInformation = LineDetectionService.calculateLineInformation(pageContents.getSortedTextPositionSequences());
         Rectangle2D mainBodyTextFrame = MainBodyTextFrameExtractionService.calculateMainBodyTextFrame(lineInformation);
-        GapInformation gapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedWords(), mainBodyTextFrame);
+        GapInformation gapInformation = GapDetectionService.findGapsInLines(pageContents.getSortedTextPositionSequences(), mainBodyTextFrame);
 
         return new PageInformation(pageContents, lineInformation, mainBodyTextFrame, gapInformation);
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -34,7 +34,22 @@ public class SimplifiedSectionTextService {
         List<SimplifiedSectionText> simplifiedText = Stream.of(simplifiedMainSectionsList, simplifiedHeadersList, simplifiedFootersList)
                 .flatMap(List::stream)
                 .collect(Collectors.toList());
-        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedText).build();
+        return SimplifiedText.builder()
+                .numberOfPages(document.getNumberOfPages())
+                .sectionTexts(simplifiedText)
+                .mainSectionNumbers(document.getAllSections()
+                                            .stream()
+                                            .map(this::getSectionNumber)
+                                            .toList())
+                .headerSectionNumbers(document.getHeaders()
+                                              .stream()
+                                              .map(this::getSectionNumber)
+                                              .toList())
+                .footerSectionNumbers(document.getFooters()
+                                              .stream()
+                                              .map(this::getSectionNumber)
+                                              .toList())
+                .build();
     }
 
 
@@ -49,4 +64,13 @@ public class SimplifiedSectionTextService {
                 .build();
     }
 
+
+    private String getSectionNumber(SemanticNode semanticNode) {
+
+        return semanticNode.getTreeId()
+                .stream()
+                .map(String::valueOf)
+                .collect(Collectors.joining("."));
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/TextRulingsClassifier.java

@@ -5,7 +5,7 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
@@ -17,9 +17,9 @@ public class TextRulingsClassifier {
     private final static double TEXT_BBOX_THRESHOLD_FACTOR = 0.15; // multiplied with text width then subtracted from word width. If ruling covers this width, it is considered as strikethrough/underline.
 
 
-    public static void classifyUnderlinedAndStrikethroughText(List<Word> words, CleanRulings cleanRulings) {
+    public static void classifyUnderlinedAndStrikethroughText(List<TextPositionSequence> words, CleanRulings cleanRulings) {
 
-        for (Word word : words) {
+        for (TextPositionSequence word : words) {
             if (word.getDir().equals(TextDirection.ZERO) || word.getDir().equals(TextDirection.HALF_CIRCLE)) {
                 handleHorizontalText(cleanRulings, word);
             } else {
@@ -29,7 +29,7 @@ public class TextRulingsClassifier {
     }
 
 
-    private static void handleVerticalText(CleanRulings cleanRulings, Word word) {
+    private static void handleVerticalText(CleanRulings cleanRulings, TextPositionSequence word) {
 
         float lowerY = (float) (word.getBBoxPdf().getMinY() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
         float upperY = (float) (word.getBBoxPdf().getMaxY() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
@@ -63,7 +63,7 @@ public class TextRulingsClassifier {
     }
 
 
-    private static void handleHorizontalText(CleanRulings cleanRulings, Word word) {
+    private static void handleHorizontalText(CleanRulings cleanRulings, TextPositionSequence word) {
 
         float leftX = (float) (word.getBBoxPdf().getMinX() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
         float rightX = (float) (word.getBBoxPdf().getMaxX() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/BlockificationPostprocessingService.java

@@ -7,6 +7,7 @@ import java.util.List;
 import java.util.ListIterator;
 import java.util.Locale;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.springframework.stereotype.Service;
 
@@ -16,8 +17,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.Data;
 
@@ -222,14 +222,14 @@ public class BlockificationPostprocessingService {
             headline = sectionIdentifier + headline;
         }
 
-        WordSequenceResult wordSequenceResult = findWordSequence(blockToSplit.getWords(), headline);
+        WordSequenceResult wordSequenceResult = findWordSequence(blockToSplit.getSequences(), headline);
         if (wordSequenceResult.inSequence.isEmpty() && !headline.equals(title)) {
-            wordSequenceResult = findWordSequence(blockToSplit.getWords(), title);
+            wordSequenceResult = findWordSequence(blockToSplit.getSequences(), title);
         }
 
         boolean modifiedBlockToSplit = false;
         if (!wordSequenceResult.inSequence.isEmpty()) {
-            blockToSplit.setWords(wordSequenceResult.inSequence);
+            blockToSplit.setSequences(wordSequenceResult.inSequence);
             blockToSplit.recalculateBBox();
             modifiedBlockToSplit = true;
         }
@@ -250,19 +250,19 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private static WordSequenceResult findWordSequence(List<Word> words, String text) {
+    private static WordSequenceResult findWordSequence(List<TextPositionSequence> textPositionSequences, String text) {
 
         String target = sanitizeString(text);
-        List<Word> inSequence = new ArrayList<>();
-        List<Word> preSequence = new ArrayList<>();
-        List<Word> postSequence = new ArrayList<>();
+        List<TextPositionSequence> inSequence = new ArrayList<>();
+        List<TextPositionSequence> preSequence = new ArrayList<>();
+        List<TextPositionSequence> postSequence = new ArrayList<>();
         StringBuilder currentSequence = new StringBuilder();
 
         if (target.isBlank()) {
             return new WordSequenceResult();
         }
 
-        for (Word sequence : words) {
+        for (TextPositionSequence sequence : textPositionSequences) {
 
             currentSequence.append(sanitizeString(sequence.toString()));
             inSequence.add(sequence);
@@ -274,10 +274,10 @@ public class BlockificationPostprocessingService {
                     int index = 0;
                     String toRemove = currentSequence.substring(0, currentSequence.length() - target.length());
 
-                    Word next = inSequence.get(index);
+                    TextPositionSequence next = inSequence.get(index);
                     while (currentSequence.length() - next.length() >= target.length()) {
 
-                        Word removed = inSequence.remove(index);
+                        TextPositionSequence removed = inSequence.remove(index);
                         currentSequence.delete(0, removed.toString().length());
                         preSequence.add(removed);
 
@@ -306,7 +306,7 @@ public class BlockificationPostprocessingService {
                 }
 
                 if (currentSequence.toString().equals(target)) {
-                    postSequence.addAll(words.subList(words.indexOf(sequence) + 1, words.size()));
+                    postSequence.addAll(textPositionSequences.subList(textPositionSequences.indexOf(sequence) + 1, textPositionSequences.size()));
                     return new WordSequenceResult(inSequence, preSequence, postSequence);
                 }
             }
@@ -316,10 +316,10 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private static SplitSequenceResult splitSequence(Word sequence, String toRemove) {
+    private static SplitSequenceResult splitSequence(TextPositionSequence sequence, String toRemove) {
 
-        Word in = null;
-        Word out;
+        TextPositionSequence in = null;
+        TextPositionSequence out;
 
         String currentSequence = sequence.toString().toLowerCase(Locale.ROOT);
         int index = currentSequence.indexOf(toRemove);
@@ -337,9 +337,9 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private static Word createSubSequence(Word sequence, int start, int end) {
+    private static TextPositionSequence createSubSequence(TextPositionSequence sequence, int start, int end) {
 
-        Word newSeq = new Word(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
+        TextPositionSequence newSeq = new TextPositionSequence(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
         newSeq.setParagraphStart(sequence.isParagraphStart());
         return newSeq;
     }
@@ -354,10 +354,10 @@ public class BlockificationPostprocessingService {
             List<TextPageBlock> mergedBlocks = new ArrayList<>();
             for (TextPageBlock textPageBlock : blocksToMerge.subList(1, blocksToMerge.size())) {
 
-                if (firstBlock != null && !firstBlock.getWords().isEmpty()) {
+                if (firstBlock != null && !firstBlock.getSequences().isEmpty()) {
 
                     if (textPageBlock.getDir() == firstBlock.getDir()) {
-                        firstBlock.addAll(textPageBlock.getWords());
+                        firstBlock.getSequences().addAll(textPageBlock.getSequences());
                         mergedBlocks.add(textPageBlock);
                     }
                 }
@@ -467,9 +467,7 @@ public class BlockificationPostprocessingService {
 
     private static String sanitizeString(String text) {
 
-        return TextNormalizationUtilities.removeAllWhitespaces(text)//
-                .trim() // sometimes there are trailing empty bytes at the end of the string trim() seems to remove them
-                .toLowerCase(Locale.ENGLISH);
+        return StringUtils.deleteWhitespace(text).toLowerCase(Locale.ROOT);
     }
 
 
@@ -496,12 +494,12 @@ public class BlockificationPostprocessingService {
 
     public static class WordSequenceResult {
 
-        public List<Word> inSequence;
-        public List<Word> preSequence;
-        public List<Word> postSequence;
+        public List<TextPositionSequence> inSequence;
+        public List<TextPositionSequence> preSequence;
+        public List<TextPositionSequence> postSequence;
 
 
-        public WordSequenceResult(List<Word> inSequence, List<Word> preSequence, List<Word> postSequence) {
+        public WordSequenceResult(List<TextPositionSequence> inSequence, List<TextPositionSequence> preSequence, List<TextPositionSequence> postSequence) {
 
             this.inSequence = inSequence;
             this.preSequence = preSequence;
@@ -522,7 +520,7 @@ public class BlockificationPostprocessingService {
 
     }
 
-    public record SplitSequenceResult(Word in, Word out) {
+    public record SplitSequenceResult(TextPositionSequence in, TextPositionSequence out) {
 
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -14,7 +14,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
 
 import lombok.RequiredArgsConstructor;
@@ -30,7 +30,7 @@ public class DocstrumBlockificationService {
     static final float THRESHOLD = 1f;
 
 
-    public ClassificationPage blockify(List<Word> textPositions,
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions,
                                        CleanRulings rulings,
                                        boolean xyOrder,
                                        LayoutDebugLayer visualizations,
@@ -72,16 +72,16 @@ public class DocstrumBlockificationService {
         List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
         zones.forEach(zone -> {
 
-            List<Word> words = new ArrayList<>();
+            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
             zone.getLines()
                     .forEach(line -> {
                         line.getWords()
                                 .forEach(word -> {
-                                    words.add(new Word(word.getTextPositions(), word.getPage()));
+                                    textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), word.getPage()));
                                 });
                     });
 
-            abstractPageBlocks.add(buildTextBlock(words, 0));
+            abstractPageBlocks.add(buildTextBlock(textPositionSequences, 0));
         });
 
         return abstractPageBlocks;
@@ -102,7 +102,7 @@ public class DocstrumBlockificationService {
             }
             TextPageBlock current = (TextPageBlock) block;
 
-            if (previous != null && !previous.getWords().isEmpty()) {
+            if (previous != null && !previous.getSequences().isEmpty()) {
 
                 if (current.getDir() != previous.getDir() || usedRulings.lineBetween(current, previous)) {
                     previous = current;
@@ -182,8 +182,8 @@ public class DocstrumBlockificationService {
 
     private TextPageBlock combineBlocksAndResetIterator(TextPageBlock previous, TextPageBlock current, ListIterator<AbstractPageBlock> itty, boolean toDuplicate) {
 
-        previous.addAll(current.getWords());
-        previous = buildTextBlock(previous.getWords(), 0);
+        previous.getSequences().addAll(current.getSequences());
+        previous = buildTextBlock(previous.getSequences(), 0);
         previous.setToDuplicate(toDuplicate);
         if (current.getClassification() != null && previous.getClassification() == null) {
             previous.setClassification(current.getClassification());
@@ -283,8 +283,8 @@ public class DocstrumBlockificationService {
                 if (current.getDir() == inner.getDir() && current.intersects(inner, yThreshold, xThreshold)) {
 
                     boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
-                    current.addAll(inner.getWords());
-                    current = buildTextBlock(current.getWords(), 0);
+                    current.getSequences().addAll(inner.getSequences());
+                    current = buildTextBlock(current.getSequences(), 0);
 
                     current.setToDuplicate(toDuplicate);
                     blocks.set(i, null);
@@ -301,7 +301,7 @@ public class DocstrumBlockificationService {
     }
 
 
-    public static TextPageBlock buildTextBlock(List<Word> wordBlockList, int indexOnPage) {
+    public static TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
         return new TextPageBlock(wordBlockList);
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocuMineBlockificationService.java

@@ -16,14 +16,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 @SuppressWarnings("all")
 @Service
 public class DocuMineBlockificationService {
 
     static final float THRESHOLD = 1f;
-    public static final double FONT_SIZE_CHANGE_RATIO = 0.15;
 
     Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);
 
@@ -37,9 +36,9 @@ public class DocuMineBlockificationService {
      * @param cleanRulings  All rulings on a page
      * @return Page object that contains the Textblock and text statistics.
      */
-    public ClassificationPage blockify(List<Word> textPositions, CleanRulings cleanRulings) {
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, CleanRulings cleanRulings) {
 
-        List<Word> chunkWords = new ArrayList<>();
+        List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractPageBlock> textPageBlocks = new ArrayList<>();
 
         CleanRulings usedRulings = cleanRulings.withoutTextRulings();
@@ -48,11 +47,11 @@ public class DocuMineBlockificationService {
         double maxX = 0;
         double minY = 1000;
         double maxY = 0;
-        Word prev = null;
+        TextPositionSequence prev = null;
 
         boolean wasSplitted = false;
         Double splitX1 = null;
-        for (Word word : textPositions) {
+        for (TextPositionSequence word : textPositions) {
 
             boolean lineSeparation = prev != null && word.getYDirAdj() - prev.getMaxYDirAdj() > Math.min(word.getHeight(), prev.getHeight()) * 1.1;
             boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
@@ -61,7 +60,12 @@ public class DocuMineBlockificationService {
             boolean newLineAfterSplit = prev != null && word.getYDirAdj() != prev.getYDirAdj() && wasSplitted && splitX1 != word.getXDirAdj();
             boolean isSplitByRuling = prev != null && usedRulings.lineBetween(prev, word);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 && isFontChange(word, prev);
+            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 //
+                                                && (word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold")
+                                                    //
+                                                    || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold")
+                                                    || Math.abs(prev.getFontSize() - word.getFontSize()) >= 1
+                                                    || Math.abs(word.getTextHeight() - prev.getTextHeight()) > 0.8);
 
             Matcher matcher = pattern.matcher(chunkWords.stream()
                                                       .collect(Collectors.joining(" ")).toString());
@@ -124,15 +128,6 @@ public class DocuMineBlockificationService {
     }
 
 
-    private static boolean isFontChange(Word word, Word prev) {
-
-        return word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold")
-               || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold")
-               || Math.abs(prev.getFontSize() - word.getFontSize()) >= FONT_SIZE_CHANGE_RATIO * Math.min(prev.getFontSize(), word.getFontSize())
-               || Math.abs(word.getTextHeight() - prev.getTextHeight()) >= FONT_SIZE_CHANGE_RATIO * Math.min(prev.getTextHeight(), word.getTextHeight());
-    }
-
-
     public void mergeblocks(ClassificationPage page, CleanRulings usedRulings, float xThreshold, float yThreshold) {
 
         var blocks = page.getTextBlocks();
@@ -175,8 +170,8 @@ public class DocuMineBlockificationService {
                         .equals(inner.getClassification()))) {
 
                     boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
-                    current.addAll(inner.getWords());
-                    current = buildTextBlock(current.getWords(), 0);
+                    current.getSequences().addAll(inner.getSequences());
+                    current = buildTextBlock(current.getSequences(), 0);
                     current.setClassification(inner.getClassification());
                     current.setToDuplicate(toDuplicate);
                     blocks.set(i, null);
@@ -199,7 +194,7 @@ public class DocuMineBlockificationService {
     }
 
 
-    public static TextPageBlock buildTextBlock(List<Word> wordBlockList, int indexOnPage) {
+    public static TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
         return new TextPageBlock(wordBlockList);
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java

@@ -11,7 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
 
 @SuppressWarnings("all")
@@ -30,20 +30,20 @@ public class RedactManagerBlockificationService {
      * @param visualizations
      * @return Page object that contains the Textblock and text statistics.
      */
-    public ClassificationPage blockify(List<Word> textPositions, CleanRulings cleanRulings, LayoutDebugLayer visualizations) {
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, CleanRulings cleanRulings, LayoutDebugLayer visualizations) {
 
         CleanRulings usedRulings = cleanRulings.withoutTextRulings();
 
         int indexOnPage = 0;
-        List<Word> chunkWords = new ArrayList<>();
+        List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractPageBlock> chunkBlockList = new ArrayList<>();
 
         double minX = 1000, maxX = 0, minY = 1000, maxY = 0;
-        Word prev = null;
+        TextPositionSequence prev = null;
 
         boolean wasSplitted = false;
         Double splitX1 = null;
-        for (Word word : textPositions) {
+        for (TextPositionSequence word : textPositions) {
 
             boolean lineSeparation = word.getYDirAdj() - maxY > word.getHeight() * 1.25;
             boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
@@ -161,6 +161,7 @@ public class RedactManagerBlockificationService {
         }
         if (!textPositions.isEmpty()) {
             visualizations.addTextBlockVisualizations(chunkBlockList.stream()
+                                                              .map(tb -> (TextPageBlock) tb)
                                                               .toList(), textPositions.get(0).getPage());
         }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClarifyndClassificationService.java

@@ -10,6 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 
 import lombok.RequiredArgsConstructor;
@@ -22,7 +23,7 @@ public class ClarifyndClassificationService {
 
     public void classifyDocument(ClassificationDocument document) {
 
-        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getHigherThanMostPopular();
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
@@ -34,10 +35,7 @@ public class ClarifyndClassificationService {
     }
 
 
-    private void classifyPage(HeadlineClassificationService headlineClassificationService,
-                              ClassificationPage page,
-                              ClassificationDocument document,
-                              List<Double> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Double> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
@@ -47,11 +45,7 @@ public class ClarifyndClassificationService {
     }
 
 
-    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
-                               TextPageBlock textBlock,
-                               ClassificationPage page,
-                               ClassificationDocument document,
-                               List<Double> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Double> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
@@ -59,58 +53,63 @@ public class ClarifyndClassificationService {
             headlineClassificationService.setLastHeadlineFromOutline(textBlock);
             return;
         }
-        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
-                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
-                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
-            return;
-        }
         if (document.getFontSizeCounter().getMostPopular() == null) {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
             return;
         }
-        if (page.getPageNumber() == 1 //
-            && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
-                && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
-            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
-                headlineClassificationService.classifyHeadline(textBlock, headlineType);
-                document.setHeadlines(true);
-            }
-        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
-                   && PositionUtils.getApproxLineCount(textBlock) < 4.9
-                   && (textBlock.getMostPopularWordStyle().equals("bold")
-                       || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
-                          && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-
-            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
-            headlineClassificationService.classifyHeadline(textBlock, headlineType);
-            document.setHeadlines(true);
-        } else if (!textBlock.getText().startsWith("Figure ")
-                   && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
-                   && textBlock.getMostPopularWordStyle().equals("bold")
-                   && !document.getFontStyleCounter().getMostPopular().equals("bold")
-                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-
-            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
-            headlineClassificationService.classifyHeadline(textBlock, headlineType);
-            document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
-                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
-                   && textBlock.getMostPopularWordStyle().equals("bold")
-                   && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
-                   && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular())
-                   && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular())
-                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
-                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
-                   && textBlock.getMostPopularWordStyle().equals("italic")
-                   && !document.getFontStyleCounter().getMostPopular().equals("italic")
-                   && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
+                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
+                .size() == 1)) {
+            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
+                textBlock.setClassification(PageBlockType.TITLE);
+            }
+        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
+                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
+                .getCountPerValue()
+                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
+                .get(0)
+                .getTextPositions()
+                .get(0)
+                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+
+            for (int i = 1; i <= headlineFontSizes.size(); i++) {
+                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
+                    PageBlockType headlineType = PageBlockType.getHeadlineType(i);
+                    headlineClassificationService.classifyHeadline(textBlock, headlineType);
+                    document.setHeadlines(true);
+                }
+            }
+        } else if (!textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
+                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+                .get(0)
+                .getTextPositions()
+                .get(0)
+                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+            PageBlockType headlineType = PageBlockType.getHeadlineType(headlineFontSizes.size() + 1);
+            headlineClassificationService.classifyHeadline(textBlock, headlineType);
+            document.setHeadlines(true);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
+                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
+                .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
+                .getMostPopular()
+                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
             textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
             textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClassificationPatterns.java

@@ -1,33 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import java.util.regex.Pattern;
-
-public class ClassificationPatterns {
-
-    public static final Pattern HEADLINE_WITH_2_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s(?:14C)?\\s*[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN = Pattern.compile("^([0-9]\\.)\\s[a-z][0-9a-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern AT_LEAST_3_CHARS_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern HEADLINE_PATTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
-
-    public static final Pattern AMOUNT_PATTERN = Pattern.compile(
-            "^\\s*\\d+(?:\\.\\d+)?\\s*(?:ml|ul|μl|l|ug|μg|g|kg|mg|cm|cm2|cm3|mm|mm2|mm3|km|km2|m|m2|m3|lb|oz|ppm|dpm|days|weeks|months|%|f|ppb)\\b",
-            Pattern.CASE_INSENSITIVE);
-
-
-
-    public static final Pattern TABLE_OR_FIGURE_HEADLINE_PATTERN = Pattern.compile(
-            "^\\s*(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
-            Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern TABLE_MID_SENTENCE_PATTERN = Pattern.compile(
-            "(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
-            Pattern.CASE_INSENSITIVE);
-
-    public static final Pattern ALPHANUMERIC = Pattern.compile("[a-zA-Z0-9]");
-
-    public static final Pattern NUMERIC = Pattern.compile("[0-9]+");
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClassificationService.java

@@ -1,62 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import java.util.Map;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ClassificationService {
-
-    DocuMineBlockificationService docuMineBlockificationService;
-    BodyTextFrameService bodyTextFrameService;
-    TableOfContentsClassificationService tableOfContentsClassificationService;
-    RedactManagerClassificationService redactManagerClassificationService;
-    ClarifyndClassificationService clarifyndClassificationService;
-    DocuMineClassificationService docuMineClassificationService;
-    HeaderFooterClassificationService headerFooterClassificationService;
-
-
-    public void classify(ClassificationDocument document, LayoutParsingType layoutParsingType, Map<String, String> identifier) {
-
-        log.info("Calculating BodyTextFrame for {}", identifier);
-        bodyTextFrameService.setBodyTextFrames(document, layoutParsingType);
-        for (ClassificationPage page : document.getPages()) {
-            document.getLayoutDebugLayer().addCleanRulingVisualization(page.getCleanRulings(), page.getPageNumber());
-        }
-        log.info("Classify TextBlocks for {}", identifier);
-
-        headerFooterClassificationService.classifyHeadersAndFooters(document);
-
-        tableOfContentsClassificationService.classifyTableOfContents(document);
-
-        switch (layoutParsingType) {
-            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
-                    redactManagerClassificationService.classifyDocument(document);
-            case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(document);
-            case CLARIFYND -> clarifyndClassificationService.classifyDocument(document);
-        }
-
-        if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
-            for (ClassificationPage page : document.getPages()) {
-                docuMineBlockificationService.mergeblocks(page, page.getCleanRulings().withoutTextRulings(), 0, 10);
-            }
-        }
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/DocuMineClassificationService.java

@@ -1,21 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.classification;
 
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.ALPHANUMERIC;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AMOUNT_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AT_LEAST_3_CHARS_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_PATTERN_WITH_SLASHES;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_2_IDENTIFER_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_MID_SENTENCE_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_OR_FIGURE_HEADLINE_PATTERN;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.springframework.stereotype.Service;
 
@@ -23,328 +11,145 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.AbstractBlockOnPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.HeaderFooterDetection;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DocuMineClassificationService {
 
-    public static final int SEPARATION_THRESHOLD = 10; // if the min distance between a textblock and all its surrounding blocks, the regexes can be more lenient.
-    public static final int SURROUNDING_BLOCKS_RADIUS = 3; // number of surrounding blocks before and after the current textblock to be tested
-
-    ListItemClassificationService listItemClassificationService;
+    private static final Pattern HEADLINE_WITH_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern AT_LEAST_3_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern HEADLINE_PATTTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
 
 
     public void classifyDocument(ClassificationDocument document) {
 
-        List<Double> headlineFontSizes = buildHeadlineFontSizes(document);
-        List<AbstractBlockOnPage> blocks = buildBlocksPerPage(document);
-        log.debug("Headline FontSizes are: {}", headlineFontSizes);
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getHigherThanMostPopular();
+
+        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
         HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();
 
-        for (int i = 0; i < blocks.size(); i++) {
-            AbstractBlockOnPage block = blocks.get(i);
-            document.getLayoutDebugLayer().addTextBlockVisualizations(block.page().getTextBlocks(), block.page().getPageNumber());
-            classifyBlock(headlineClassificationService, i, blocks, document, headlineFontSizes);
+        for (ClassificationPage page : document.getPages()) {
+            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
         }
+    }
 
+
+    private void classifyPage(HeadlineClassificationService headlineClassificationService,
+                              ClassificationPage page,
+                              ClassificationDocument document,
+                              List<Double> headlineFontSizes) {
+
+        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, page, document, headlineFontSizes);
+            }
+        }
     }
 
 
     private void classifyBlock(HeadlineClassificationService headlineClassificationService,
-                               int currentIndex,
-                               List<AbstractBlockOnPage> allBlocks,
+                               TextPageBlock textBlock,
+                               ClassificationPage page,
                                ClassificationDocument document,
                                List<Double> headlineFontSizes) {
 
-        TextPageBlock textBlock;
-        if (allBlocks.get(currentIndex).block() instanceof TextPageBlock block) {
-            textBlock = block;
-        } else {
-            return;
-        }
-        ClassificationPage page = allBlocks.get(currentIndex).page();
-        List<AbstractPageBlock> surroundingBlocks = getSurroundingBlocksOnPage(currentIndex, allBlocks);
-
         log.debug("headlineFontSizes: {}", headlineFontSizes);
         var bodyTextFrame = page.getBodyTextFrame();
 
-        Matcher headlineWith2IdentifierMatcher = HEADLINE_WITH_2_IDENTIFER_PATTERN.matcher(textBlock.toString());
-        Matcher atLeast3Matcher = AT_LEAST_3_CHARS_PATTERN.matcher(textBlock.toString());
-        Matcher headlineWithSlashesMatcher = HEADLINE_PATTERN_WITH_SLASHES.matcher(textBlock.toString());
-        Matcher amountMatcher = AMOUNT_PATTERN.matcher(textBlock.toString());
-        Matcher tableOrFigureMatcher = TABLE_OR_FIGURE_HEADLINE_PATTERN.matcher(textBlock.toString());
-        Matcher tableMidSentenceMatcher = TABLE_MID_SENTENCE_PATTERN.matcher(textBlock.toString());
-        Matcher headlineWithSingleIdentifierMatcher = HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN.matcher(textBlock.toString());
-        boolean isAtLeast3Characters = atLeast3Matcher.reset().find();
-        boolean headlineWithSlashesMatches = headlineWithSlashesMatcher.reset().matches();
-        boolean isAmount = amountMatcher.reset().find();
-        int charCount = countChars(textBlock);
-
-        boolean enoughChars = charCount > textBlock.getText().length() * 0.5;
-
-        List<ListIdentifier> listIdentifiers = listItemClassificationService.findConfirmedListIdentifiers(currentIndex, allBlocks);
-        document.getLayoutDebugLayer().addListIdentifiers(listIdentifiers);
+        Matcher headlineWithIdentifierMatcher = HEADLINE_WITH_IDENTIFER_PATTERN.matcher(textBlock.toString());
+        Matcher atLeast3Matcher = AT_LEAST_3_PATTERN.matcher(textBlock.toString());
+        Matcher headlineWithSlashesMatcher = HEADLINE_PATTTERN_WITH_SLASHES.matcher(textBlock.toString());
 
         if (textBlock.getClassification() != null && textBlock.getClassification().isHeadline()) {
             headlineClassificationService.setLastHeadlineFromOutline(textBlock);
             return;
         }
-        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
-                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
-                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
-            return;
-        }
         if (document.getFontSizeCounter().getMostPopular() == null) {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
             return;
         }
-        if (textBlock.getText().length() > 5
-            && greaterOrEqualFontThanDocumentAverage(textBlock, document)
-            && PositionUtils.getApproxLineCount(textBlock) < 5.9
-            && ((textBlock.getMostPopularWordStyle().contains("bold") || textBlock.isUnderlined())//
-                && Character.isDigit(textBlock.toString().charAt(0)) //
-                && isAtLeast3Characters //
-                && !textBlock.toString().contains(":") //
-                || textBlock.toString().startsWith("APPENDIX") //
-                || textBlock.toString().startsWith("FIGURE") //
-                || textBlock.toString().startsWith("Continued TABLE") //
-                || textBlock.toString().startsWith("TABLE"))
-            && !textBlock.toString().endsWith(":")
-            && isAtLeast3Characters
-            && !isAmount
-            && enoughChars) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) //
+            || (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) //
+                && (document.getFontSizeCounter().getMostPopular() == null //
+                    || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular()))) {
+            textBlock.setClassification(PageBlockType.HEADER);
 
-            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
-        } else if (isAllCaps(textBlock)
-                   && ALPHANUMERIC.matcher(Character.toString(textBlock.getText().charAt(0))).matches()
-                   && hasSeparation(textBlock, surroundingBlocks)
-                   && textBlock.getText().length() > 5
-                   && isAtLeast3Characters
-                   && !isAmount
-                   && enoughChars
-                   && !textBlock.toString().contains(":")
-                   && !textBlock.toString().endsWith(".")
-                   && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                   || (PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                                                          textBlock,
+                                                          page.getRotation())
+                       && (document.getFontSizeCounter().getMostPopular()
+                           == null
+                           || textBlock.getHighestFontSize()
+                              <= document.getFontSizeCounter()
+                                      .getMostPopular()))
+                   || HeaderFooterDetection.isLikelyFooter(textBlock, document, page)) {
+            textBlock.setClassification(PageBlockType.FOOTER);
+        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
+                                                 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
+            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
+                textBlock.setClassification(PageBlockType.TITLE);
+            }
+        } else if (textBlock.getText().length() > 5
+                   && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular()
+                       || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular())
+                   && PositionUtils.getApproxLineCount(textBlock) < 5.9
 
-            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
-        } else if (headlineWith2IdentifierMatcher.reset().find()
+                   && (textBlock.getMostPopularWordStyle().contains("bold")
+                       && Character.isDigit(textBlock.toString().charAt(0))
+                       && atLeast3Matcher.reset().find()
+                       && !textBlock.toString().contains(":") //
+                       || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT))
+                          && atLeast3Matcher.reset().find()
+                          && !textBlock.toString().contains(":")
+                          && !textBlock.toString().startsWith("(")//
+                       || textBlock.toString().startsWith("APPENDIX") //
+                       || textBlock.toString().startsWith("FIGURE") //
+                       || textBlock.toString().startsWith("Continued TABLE") //
+                       || textBlock.toString().startsWith("TABLE"))
+                   && !textBlock.toString().endsWith(":")
+                   && atLeast3Matcher.reset().find()) {
+            PageBlockType headlineType = PageBlockType.getHeadlineType(1);
+            headlineClassificationService.classifyHeadline(textBlock, headlineType);
+            document.setHeadlines(true);
+
+        } else if (headlineWithIdentifierMatcher.reset().find()
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && isAtLeast3Characters
-                   && !headlineWithSlashesMatches
-                   && !isAmount) {
-
-            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
-        } else if (hasSeparation(textBlock, surroundingBlocks)//
-                   && greaterOrEqualFontThanPageAverage(textBlock, page)//
-                   && PositionUtils.getApproxLineCount(textBlock) < 2.9//
-                   && (tableOrFigureMatcher.reset().find() || (headlineWithSingleIdentifierMatcher.reset().find() && listIdentifiers.isEmpty())) //
-                   && tableMidSentenceMatcher.reset().results()
-                              .count() <= 1 //
-                   && !isAmount//
-                   && !headlineWithSlashesMatches) {
-
-            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
-//        } else if (textBlock.getMostPopularWordFont().contains("bold")
-//                   && greaterOrEqualFontThanPageAverage(textBlock, page)
-//                   && textBlock.getWords().size() <= 6
-//                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
-//                   && isAtLeast3Characters
-//                   && charCount > textBlock.getText().length() * 0.75
-//                   && !textBlock.getText().contains(":")
-//                   && textBlock.getWidth() < page.getBodyTextFrame().getWidth() * 0.7) {
-//
-//            setAsHeadline(headlineClassificationService, textBlock, document, headlineFontSizes);
-        } else if (!listIdentifiers.isEmpty()) {
-
-            textBlock.setClassification(PageBlockType.LIST_ITEM);
+                   && atLeast3Matcher.reset().find()
+                   && !headlineWithSlashesMatcher.reset().matches()) {
+            PageBlockType headlineType = PageBlockType.getHeadlineType(2);
+            headlineClassificationService.classifyHeadline(textBlock, headlineType);
+            document.setHeadlines(true);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
-
             textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular())
                    && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular())
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
-
             textBlock.setClassification(PageBlockType.PARAGRAPH);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                    && textBlock.getMostPopularWordStyle().equals("italic")
                    && !document.getFontStyleCounter().getMostPopular().equals("italic")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
-
             textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
         } else {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
         }
     }
 
-
-    private int countChars(TextPageBlock textBlock) {
-
-        int count = 0;
-
-        for (int i = 0; i < textBlock.getText().length(); i++) {
-            if (Character.isAlphabetic(textBlock.getText().charAt(i))) {
-                count++;
-            }
-        }
-        return count;
-    }
-
-
-    private static boolean greaterOrEqualFontThanPageAverage(TextPageBlock textBlock, ClassificationPage page) {
-
-        return textBlock.getMostPopularWordHeight() >= page.getTextHeightCounter().getMostPopular() //
-               || textBlock.getMostPopularWordFontSize() >= page.getFontSizeCounter().getMostPopular();
-    }
-
-
-    private static boolean greaterOrEqualFontThanDocumentAverage(TextPageBlock textBlock, ClassificationDocument document) {
-
-        return textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() //
-               || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular();
-    }
-
-
-    private static boolean isAllCaps(TextPageBlock textBlock) {
-
-        return textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT));
-    }
-
-
-    private boolean hasSeparation(TextPageBlock textBlock, List<AbstractPageBlock> surroundingBlocks) {
-
-        return surroundingBlocks.stream()
-                .allMatch(surroundingBlock -> calculateSeparation(textBlock, surroundingBlock) > Math.pow(SEPARATION_THRESHOLD, 2));
-    }
-
-
-    private double calculateMinSeparation(TextPageBlock textBlock, List<AbstractPageBlock> surroundingBlocks) {
-
-        return surroundingBlocks.stream()
-                .mapToDouble(surroundingBlock -> calculateSeparation(textBlock, surroundingBlock))
-                .min().orElse(Double.MAX_VALUE);
-    }
-
-
-    private static double calculateSeparation(TextPageBlock textBlock, AbstractPageBlock surroundingBlock) {
-
-        return Math.pow(surroundingBlock.horizontalDistance(textBlock), 2) + Math.pow(surroundingBlock.verticalDistance(textBlock), 2);
-    }
-
-
-    private static void setAsHeadline(HeadlineClassificationService headlineClassificationService,
-                                      TextPageBlock textBlock,
-                                      ClassificationDocument document,
-                                      List<Double> headlineFontSizes) {
-
-        PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
-        headlineClassificationService.classifyHeadline(textBlock, headlineType);
-        document.setHeadlines(true);
-    }
-
-
-    private List<AbstractBlockOnPage> buildBlocksPerPage(ClassificationDocument document) {
-
-        List<AbstractBlockOnPage> blocks = new ArrayList<>();
-        for (ClassificationPage page : document.getPages()) {
-            for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
-                if (abstractPageBlock instanceof TextPageBlock textBlock) {
-                    if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER) //
-                                                                  || textBlock.getClassification().equals(PageBlockType.FOOTER))) {
-                        continue;
-                    }
-                    blocks.add(new AbstractBlockOnPage(textBlock, page));
-                }
-            }
-        }
-        return blocks;
-    }
-
-
-    private static List<Double> buildHeadlineFontSizes(ClassificationDocument document) {
-
-        if (document.getFontSizeCounter().getCountPerValue().size() <= 6) {
-            return document.getFontSizeCounter().getValuesInReverseOrder();
-        }
-
-        List<Map.Entry<Double, Integer>> sortedEntries = new ArrayList<>(document.getFontSizeCounter().getCountPerValue().entrySet());
-        sortedEntries.sort(Map.Entry.comparingByKey());
-
-        int totalCount = sortedEntries.stream()
-                .mapToInt(Map.Entry::getValue).sum();
-
-        int cumulativeCount = 0;
-        Iterator<Map.Entry<Double, Integer>> iterator = sortedEntries.iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<Double, Integer> entry = iterator.next();
-            cumulativeCount += entry.getValue();
-            if (cumulativeCount > totalCount * 0.3) {
-                break; // We've filtered the bottom 30%, so stop.
-            }
-            iterator.remove();
-        }
-
-        if (sortedEntries.size() < 6) {
-            return document.getFontSizeCounter().getValuesInReverseOrder();
-        }
-        int clusterSize = Math.max(1, sortedEntries.size() / 6);
-
-        List<List<Double>> clusters = new ArrayList<>();
-        for (int i = 0; i < 6; i++) {
-            clusters.add(new ArrayList<>());
-        }
-
-        for (int i = 0; i < sortedEntries.size(); i++) {
-            int clusterIndex = Math.min(i / clusterSize, 5);
-            clusters.get(clusterIndex).add(sortedEntries.get(i).getKey());
-        }
-
-        return clusters.stream()
-                .map(cluster -> cluster.stream()
-                        .mapToDouble(d -> d).average()
-                        .orElseThrow())
-                .sorted(Comparator.reverseOrder())
-                .toList();
-    }
-
-
-    private List<AbstractPageBlock> getSurroundingBlocksOnPage(int originalIndex, List<AbstractBlockOnPage> textBlocks) {
-
-        int start = Math.max(originalIndex - SURROUNDING_BLOCKS_RADIUS, 0);
-        int end = Math.min(originalIndex + SURROUNDING_BLOCKS_RADIUS, textBlocks.size());
-        List<AbstractPageBlock> surroundingBlocks = new ArrayList<>(2 * SURROUNDING_BLOCKS_RADIUS);
-        for (int i = start; i < end; i++) {
-            if (i == originalIndex) {
-                continue;
-            }
-            if (textBlocks.get(i).block().getText().length() <= 1) {
-                continue;
-            }
-            if (!textBlocks.get(i).page().equals(textBlocks.get(originalIndex).page())) {
-                continue;
-            }
-            surroundingBlocks.add(textBlocks.get(i).block());
-        }
-        return surroundingBlocks;
-    }
-
-}
-
-
-
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/HeaderFooterClassificationService.java

@@ -1,55 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class HeaderFooterClassificationService {
-
-    public void classifyHeadersAndFooters(ClassificationDocument document) {
-
-        for (ClassificationPage page : document.getPages()) {
-            for (AbstractPageBlock pageBlock : page.getTextBlocks()) {
-                if (pageBlock instanceof TextPageBlock textBlock) {
-                    classifyBlock(document, page, textBlock);
-                }
-            }
-        }
-
-    }
-
-
-    private static void classifyBlock(ClassificationDocument document, ClassificationPage page, TextPageBlock textBlock) {
-
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-            || PositionUtils.isOverBodyTextFrame(page.getBodyTextFrame(), textBlock, page.getRotation()) && smallerFontThanDocAverage(document, textBlock)) {
-
-            textBlock.setClassification(PageBlockType.HEADER);
-
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                   || PositionUtils.isUnderBodyTextFrame(page.getBodyTextFrame(), textBlock, page.getRotation()) && smallerFontThanDocAverage(document, textBlock)) {
-
-            textBlock.setClassification(PageBlockType.FOOTER);
-        }
-    }
-
-
-    private static boolean smallerFontThanDocAverage(ClassificationDocument document, TextPageBlock textBlock) {
-
-        return document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/HeadlineClassificationService.java

@@ -2,10 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 
 import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.getHeadlineNumber;
 
-import java.util.List;
-
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import lombok.Getter;
@@ -19,7 +16,6 @@ public class HeadlineClassificationService {
     PageBlockType originalClassifiedBlockType;
     TextPageBlock lastHeadlineFromOutline;
 
-
     public void setLastHeadlineFromOutline(TextPageBlock lastHeadlineFromOutline) {
 
         this.lastHeadlineFromOutline = lastHeadlineFromOutline;
@@ -29,62 +25,28 @@ public class HeadlineClassificationService {
 
     public void classifyHeadline(TextPageBlock textBlock, PageBlockType initialHeadlineType) {
 
+        TextPageBlock lastHeadline = getLastHeadline();
+        TextPageBlock lastHeadlineFromOutline = getLastHeadlineFromOutline();
+        PageBlockType originalClassifiedBlockType = getOriginalClassifiedBlockType();
         PageBlockType finalHeadlineType = initialHeadlineType;
 
         if (lastHeadline != null) {
 
-            finalHeadlineType = decideOnClassification(textBlock, initialHeadlineType);
-        }
+            if (lastHeadline.equals(lastHeadlineFromOutline)) {
 
-        lastHeadline = textBlock;
-        originalClassifiedBlockType = initialHeadlineType;
-        textBlock.setClassification(finalHeadlineType);
-    }
+                finalHeadlineType = PageBlockType.getHeadlineType(getHeadlineNumber(lastHeadline.getClassification()) + 1);
 
+            } else if (originalClassifiedBlockType != null && lastHeadline.getClassification() != originalClassifiedBlockType) {
 
-    private PageBlockType decideOnClassification(TextPageBlock textBlock, PageBlockType initialHeadlineType) {
-
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText(textBlock.getText());
-        TextPageBlock lastHeadlineFromOutline = getLastHeadlineFromOutline();
-        PageBlockType originalClassifiedBlockType = getOriginalClassifiedBlockType();
-
-        if (!identifier.isEmpty()) {
-            return PageBlockType.getHeadlineType(identifier.level());
-        }
-
-        if (lastHeadline.equals(lastHeadlineFromOutline) && lastHeadline.getMostPopularWordFontSize() >= textBlock.getMostPopularWordFontSize()) {
-
-            return PageBlockType.getHeadlineType(getHeadlineNumber(lastHeadline.getClassification()) + 1);
-
-        } else if (originalClassifiedBlockType != null && lastHeadline.getClassification() != originalClassifiedBlockType) {
-
-            return adjustInitialLevelToLastHeadlineLevel(initialHeadlineType);
-        }
-        return initialHeadlineType;
-    }
-
-
-    private PageBlockType adjustInitialLevelToLastHeadlineLevel(PageBlockType initialHeadlineType) {
-
-        int difference = getHeadlineNumber(originalClassifiedBlockType) - getHeadlineNumber(lastHeadline.getClassification());
-        return PageBlockType.getHeadlineType(Math.max(1, getHeadlineNumber(initialHeadlineType) - difference));
-    }
-
-
-    public static PageBlockType headlineClassByFontSize(TextPageBlock textBlock, List<Double> fontSizeGroups) {
-
-        List<Double> distances = fontSizeGroups.stream()
-                .map(fontSize -> Math.abs(fontSize - textBlock.getMostPopularWordFontSize()))
-                .toList();
-        double min = Double.MAX_VALUE;
-        int argMin = -1;
-        for (int i = 0; i < distances.size(); i++) {
-            if (distances.get(i) < min) {
-                min = distances.get(i);
-                argMin = i;
+                PageBlockType lastHeadlineType = lastHeadline.getClassification();
+                int difference = getHeadlineNumber(originalClassifiedBlockType) - getHeadlineNumber(lastHeadlineType);
+                finalHeadlineType = PageBlockType.getHeadlineType(getHeadlineNumber(initialHeadlineType) - difference);
             }
         }
-        return PageBlockType.getHeadlineType(argMin);
+
+        setOriginalClassifiedBlockType(initialHeadlineType);
+        textBlock.setClassification(finalHeadlineType);
+        setLastHeadline(textBlock);
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ListItemClassificationService.java

@@ -1,99 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.AbstractBlockOnPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-
-@Service
-public class ListItemClassificationService {
-
-    public static final int LIST_IDENTIFIER_SEARCH_RADIUS = 3;
-
-
-    public List<ListIdentifier> findConfirmedListIdentifiers(int currentIndex, List<AbstractBlockOnPage> allBlocks) {
-
-        List<ListIdentifier> listIdentifiers = extractListIdentifiers(allBlocks.get(currentIndex));
-        if (listIdentifiers.isEmpty()) {
-            return Collections.emptyList();
-        }
-        if (listIdentifiers.size() > 1 && ListIdentifier.isInOrder(listIdentifiers)) {
-            return listIdentifiers;
-        }
-
-        int start = Math.max(0, currentIndex - LIST_IDENTIFIER_SEARCH_RADIUS);
-        int end = Math.min(allBlocks.size(), currentIndex + LIST_IDENTIFIER_SEARCH_RADIUS);
-
-        List<ListIdentifier> identifiersBehind = new ArrayList<>();
-        if (start < currentIndex) {
-            identifiersBehind.addAll(allBlocks.subList(start, currentIndex)
-                                             .stream()
-                                             .map(this::extractListIdentifiers)
-                                             .flatMap(Collection::stream)
-                                             .toList());
-        }
-        if (!identifiersBehind.isEmpty()) {
-            listIdentifiers.add(0, identifiersBehind.get(identifiersBehind.size() - 1));
-            if (ListIdentifier.isInOrder(listIdentifiers)) {
-                return listIdentifiers;
-            }
-            listIdentifiers.remove(0);
-        }
-        List<ListIdentifier> identifiersAhead = new ArrayList<>();
-        if (currentIndex + 1 < end) {
-            identifiersAhead.addAll(allBlocks.subList(currentIndex + 1, end)
-                                            .stream()
-                                            .map(this::extractListIdentifiers)
-                                            .flatMap(Collection::stream)
-                                            .toList());
-        }
-        if (!identifiersAhead.isEmpty()) {
-            listIdentifiers.add(identifiersAhead.get(0));
-            if (ListIdentifier.isInOrder(listIdentifiers)) {
-                return listIdentifiers;
-            }
-            listIdentifiers.remove(listIdentifiers.size() - 1);
-        }
-        return Collections.emptyList();
-
-    }
-
-
-    private List<ListIdentifier> extractListIdentifiers(AbstractBlockOnPage block) {
-
-        List<ListIdentifier> result = new LinkedList<>();
-        if (block.block() instanceof TextPageBlock textBlock) {
-            List<Word> sequences = textBlock.getWords();
-            for (int i = 0; i < sequences.size(); i++) {
-
-                if (i != 0 && sequences.get(i - 1).getXDirAdj() < sequences.get(i).getXDirAdj()) {
-                    // is not the start of a line, continue
-                    continue;
-                }
-
-                Word sequence = sequences.get(i);
-                List<Word> wordsAtStartOfLine = new ArrayList<>(3);
-                int end = Math.min(sequences.size(), i + 3);
-                for (int j = i; j < end; j++) {
-                    if (sequences.get(j).intersectsYDirAdj(sequence, 2)) {
-                        wordsAtStartOfLine.add(sequences.get(j));
-                    } else {
-                        break;
-                    }
-                }
-
-                ListIdentifier.parse(wordsAtStartOfLine, block.page().getPageNumber()).ifPresent(result::add);
-            }
-        }
-        return result;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/NumberWord.java

@@ -1,7 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-
-public record NumberWord(Word word, int number) {
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/RedactManagerClassificationService.java

@@ -11,6 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 
 import lombok.RequiredArgsConstructor;
@@ -21,9 +22,10 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactManagerClassificationService {
 
+
     public void classifyDocument(ClassificationDocument document) {
 
-        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getHigherThanMostPopular();
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
@@ -35,10 +37,7 @@ public class RedactManagerClassificationService {
     }
 
 
-    private void classifyPage(HeadlineClassificationService headlineClassificationService,
-                              ClassificationPage page,
-                              ClassificationDocument document,
-                              List<Double> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Double> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
@@ -48,11 +47,7 @@ public class RedactManagerClassificationService {
     }
 
 
-    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
-                               TextPageBlock textBlock,
-                               ClassificationPage page,
-                               ClassificationDocument document,
-                               List<Double> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Double> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
@@ -60,15 +55,6 @@ public class RedactManagerClassificationService {
             headlineClassificationService.setLastHeadlineFromOutline(textBlock);
             return;
         }
-        if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER)
-                                                      || textBlock.getClassification().equals(PageBlockType.FOOTER)
-                                                      || textBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM))) {
-            return;
-        }
-        if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
-            return;
-        }
         if (document.getFontSizeCounter().getMostPopular() == null) {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
             return;
@@ -81,55 +67,66 @@ public class RedactManagerClassificationService {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
             return;
         }
-        if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
-                                          && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+            || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
+                                                                                                   || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
+            textBlock.setClassification(PageBlockType.HEADER);
 
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                   || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
+                                                                                                           || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
+            textBlock.setClassification(PageBlockType.FOOTER);
+        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
+                                                 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
-                headlineClassificationService.classifyHeadline(textBlock, headlineType);
-                document.setHeadlines(true);
+                textBlock.setClassification(PageBlockType.TITLE);
             }
         } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
                    && PositionUtils.getApproxLineCount(textBlock) < 4.9
                    && (textBlock.getMostPopularWordStyle().equals("bold")
                        || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                           && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getSequences()
+                              .get(0).getTextPositions()
+                              .get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
-            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
-            headlineClassificationService.classifyHeadline(textBlock, headlineType);
-            document.setHeadlines(true);
+            for (int i = 1; i <= headlineFontSizes.size(); i++) {
+                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
+                    PageBlockType headlineType = PageBlockType.getHeadlineType(i);
+                    headlineClassificationService.classifyHeadline(textBlock, headlineType);
+                    document.setHeadlines(true);
+                }
+            }
         } else if (!textBlock.getText().startsWith("Figure ")
                    && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-
-            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+                   && textBlock.getSequences()
+                              .get(0).getTextPositions()
+                              .get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+            PageBlockType headlineType = PageBlockType.getHeadlineType(headlineFontSizes.size() + 1);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
             document.setHeadlines(true);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
-
             textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular())
                    && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular())
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
-
             textBlock.setClassification(PageBlockType.PARAGRAPH);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                    && textBlock.getMostPopularWordStyle().equals("italic")
                    && !document.getFontStyleCounter().getMostPopular().equals("italic")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
-
             textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
-
             textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
         } else {
             textBlock.setClassification(PageBlockType.PARAGRAPH);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/TableOfContentsClassificationService.java

@@ -1,427 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.AMOUNT_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.NUMERIC;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextBlockOnPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TocNumberComparator;
-import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
-import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-public class TableOfContentsClassificationService {
-
-    private static final int MAX_PAGE_COUNT = 10; // maximum length of a toc to avoid runaway classification
-    private static final int SURROUNDING_BLOCKS_RADIUS = 10; // number of blocks to look ahead
-    private static final int MINIMUM_MATCHES = 2; // minimum cluster size
-    public static final int INTERSECTION_TOLERANCE = 2; // equality threshold for x intersection
-    public static final int DENSITY_THRESHOLD_COUNT = 10; // describes the minimum density, at least this many entries per page height are required
-
-
-    @SuppressWarnings("checkstyle:ModifiedControlVariable")
-    public void classifyTableOfContents(ClassificationDocument document) {
-
-        List<TextBlockOnPage> textBlocks = buildBlocksPerPage(document);
-
-        for (int i = 0; i < textBlocks.size(); i++) {
-            TextBlockOnPage textBlock = textBlocks.get(i);
-
-            if (!isTOCHeadline(textBlock)) {
-                continue;
-            }
-
-            int end = identifyTOCItems(i + 1, textBlocks, document);
-
-            if (end > i + 1) {
-                if (textBlock.textBlock().getClassification() == null) {
-                    textBlock.textBlock().setClassification(PageBlockType.H1);
-                }
-                i = end;
-            }
-        }
-    }
-
-
-    private int identifyTOCItems(int start, List<TextBlockOnPage> textBlocks, ClassificationDocument document) {
-
-        if (start >= textBlocks.size()) {
-            return start;
-        }
-        ClassificationPage startPage = textBlocks.get(start).page();
-        List<TextBlockOnPage> initialLookAhead = textBlocks.subList(start, Math.min(start + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()));
-        HashMap<NumberWord, TextBlockOnPage> numberToBlockLookup = new HashMap<>();
-        List<NumberWord> numbers = extractNumbers(initialLookAhead, numberToBlockLookup, document.getPages().size());
-        TocNumberFinder tocNumberFinder = new TocNumberFinder(numbers, numberToBlockLookup);
-
-        int lastCandidate = start;
-        for (int i = start; i < Math.min(lastCandidate + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()); i++) {
-
-            TextBlockOnPage textBlockOnPage = textBlocks.get(i);
-            if (textBlockOnPage.page().getPageNumber() - MAX_PAGE_COUNT > startPage.getPageNumber()) {
-                break;
-            }
-
-            if (textBlockOnPage.textBlock().getClassification() != null //
-                && textBlockOnPage.textBlock().getClassification().isHeadline() //
-                && !(textBlockOnPage.textBlock().getText().startsWith("TABLES") //
-                     || textBlockOnPage.textBlock().getText().startsWith("APPENDICES") //
-                     || textBlockOnPage.textBlock().getText().startsWith("FIGURES"))) {
-                log.debug("hit an outline headline, stop immediately.");
-                lastCandidate = i - 1;
-                break;
-            }
-
-            List<NumberWord> numbersFromBlock = extractNumbers(textBlockOnPage, numberToBlockLookup, document.getPages().size());
-
-            List<NumberWord> currentRightmostCluster = tocNumberFinder.getCurrentRightmostCluster();
-
-            if (currentRightmostCluster.size() < MINIMUM_MATCHES) {
-                log.debug("No numbers indicating a table of contents here.");
-                return start;
-            }
-
-            if (anyIntersection(currentRightmostCluster, numbersFromBlock, numberToBlockLookup)) {
-                lastCandidate = i;
-                numbersFromBlock.forEach(tocNumberFinder::add);
-            }
-        }
-
-        Set<TextBlockOnPage> blocksWithNumberInCluster = tocNumberFinder.getCurrentRightmostCluster()
-                .stream()
-                .map(numberToBlockLookup::get)
-                .collect(Collectors.toSet());
-
-        addVisualization(document.getLayoutDebugLayer(), tocNumberFinder, numberToBlockLookup, blocksWithNumberInCluster, textBlocks.get(start - 1));
-
-        int lastConfirmed = start;
-        for (int i = start; i < lastCandidate + 1; i++) {
-            TextBlockOnPage textBlockOnPage = textBlocks.get(i);
-            if (blocksWithNumberInCluster.contains(textBlockOnPage)) {
-                lastConfirmed = i;
-            }
-        }
-
-        textBlocks.subList(start, lastConfirmed + 1)
-                .stream()
-                .filter(block -> (block.textBlock().getClassification() == null || !block.textBlock().getClassification().isHeadline()))
-                .forEach(textBlockOnPage -> textBlockOnPage.textBlock().setClassification(PageBlockType.TABLE_OF_CONTENTS_ITEM));
-
-        return lastCandidate;
-    }
-
-
-    private static void addVisualization(LayoutDebugLayer layoutDebugLayer,
-                                         TocNumberFinder tocNumberFinder,
-                                         Map<NumberWord, TextBlockOnPage> lookup,
-                                         Set<TextBlockOnPage> blocksWithNumberInCluster,
-                                         TextBlockOnPage startingHeadline) {
-
-        tocNumberFinder.getCurrentRightmostCluster()
-                .stream()
-                .collect(Collectors.groupingBy(key -> lookup.get(key).page().getPageNumber()))
-                .forEach((pageNumber, number) -> layoutDebugLayer.addTocPages(number, pageNumber));
-        layoutDebugLayer.addTocBlocks(blocksWithNumberInCluster);
-        layoutDebugLayer.addTocBlocks(Set.of(startingHeadline));
-    }
-
-
-    private static boolean anyIntersection(Collection<NumberWord> numbers1, Collection<NumberWord> numbers2, Map<NumberWord, TextBlockOnPage> lookup) {
-
-        return numbers1.stream()
-                .anyMatch(numberFromCluster -> numbers2.stream()
-                        .anyMatch(numberFromBlock -> matches(numberFromBlock, numberFromCluster, lookup)));
-    }
-
-
-    private static List<NumberWord> extractNumbers(List<TextBlockOnPage> textBlocks, Map<NumberWord, TextBlockOnPage> lookup, int numberOfPages) {
-
-        List<NumberWord> blocks = new LinkedList<>();
-        for (TextBlockOnPage textBlock : textBlocks) {
-            blocks.addAll(extractNumbers(textBlock, lookup, numberOfPages));
-        }
-        return blocks;
-    }
-
-
-    private static List<NumberWord> extractNumbers(TextBlockOnPage textBlock, Map<NumberWord, TextBlockOnPage> lookup, int numberOfPages) {
-
-        List<NumberWord> blocks = new LinkedList<>();
-        TextPageBlock block = textBlock.textBlock();
-        List<Word> words = block.getWords();
-        for (int i = 0; i < words.size(); i++) {
-
-            Word word = words.get(i);
-            if (!wordIsEndOfLine(i, words)) {
-                continue;
-            }
-
-            if (AMOUNT_PATTERN.matcher(getSurroundingString(i, words)).matches()) {
-                continue;
-            }
-
-            Matcher matcher = SectionIdentifier.numericalIdentifierPattern.matcher(word.toString());
-            if (matcher.find() && matcher.group(2) != null) {
-                continue;
-            }
-
-            Matcher numberFinder = NUMERIC.matcher(word);
-            if (!numberFinder.find() || word.length() > 5) {
-                continue;
-            }
-
-            try {
-                int pageNumber = Integer.parseInt(numberFinder.group());
-                if (0 >= pageNumber || pageNumber > numberOfPages) {
-                    continue;
-                }
-                NumberWord numberWord = new NumberWord(word, pageNumber);
-                lookup.put(numberWord, textBlock);
-                blocks.add(numberWord);
-            } catch (NumberFormatException e) {
-                log.debug("That wasn't a number! Should not happen, due to numeric check beforehand.");
-            }
-        }
-        return blocks;
-    }
-
-
-    private static boolean wordIsEndOfLine(int i, List<Word> words) {
-
-        if (i == words.size() - 1) {
-            return true;
-        }
-        Word word = words.get(i);
-        Word nextWord = words.get(i + 1);
-        return !nextWord.rightOf(word);
-    }
-
-
-    private static CharSequence getSurroundingString(int i, List<Word> sequences) {
-
-        int end = Math.min(i + 5, sequences.size());
-        return sequences.subList(i, end)
-                .stream()
-                .map(Word::toString)
-                .collect(Collectors.joining(" "));
-    }
-
-
-    private static boolean matches(NumberWord number1, NumberWord number2, Map<NumberWord, TextBlockOnPage> lookup) {
-
-        if (number1.word().getDir() != number2.word().getDir()) {
-            return false;
-        }
-
-        return number1.word().intersectsXDirAdj(number2.word(), INTERSECTION_TOLERANCE);
-    }
-
-
-    private boolean isTOCHeadline(TextBlockOnPage textBlock) {
-
-        if (textBlock.textBlock().getText().length() > 50) {
-            return false;
-        }
-        String text = TextNormalizationUtilities.removeAllWhitespaces(textBlock.textBlock().getText().toLowerCase(Locale.ENGLISH));
-        return (text.contains("content") && text.length() < "content".length() + 6) //
-               || (text.contains("tableofcontent") && text.length() < "tableofcontent".length() + DENSITY_THRESHOLD_COUNT)//
-               || text.equals("tables")//
-               || text.equals("appendices")//
-               || text.equals("figures");
-    }
-
-
-    private List<TextBlockOnPage> buildBlocksPerPage(ClassificationDocument document) {
-
-        List<TextBlockOnPage> blocks = new ArrayList<>();
-        for (ClassificationPage page : document.getPages()) {
-            for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
-                if (abstractPageBlock instanceof TextPageBlock textBlock) {
-                    if (textBlock.getClassification() != null && (textBlock.getClassification().equals(PageBlockType.HEADER) //
-                                                                  || textBlock.getClassification().equals(PageBlockType.FOOTER))) {
-                        continue;
-                    }
-                    blocks.add(new TextBlockOnPage(textBlock, page));
-                }
-            }
-        }
-        return blocks;
-    }
-
-
-    private static class TocNumberFinder {
-
-        final UnionFind<NumberWord> numberClusters;
-        final HashMap<NumberWord, TextBlockOnPage> lookup;
-
-
-        TocNumberFinder(List<NumberWord> blocks, HashMap<NumberWord, TextBlockOnPage> lookup) {
-
-            this.numberClusters = new UnionFind<>(new HashSet<>(blocks));
-            for (int i = 0; i < blocks.size(); i++) {
-                for (int j = i + 1; j < blocks.size(); j++) {
-                    if (matches(blocks.get(i), blocks.get(j), lookup)) {
-                        numberClusters.union(blocks.get(i), blocks.get(j));
-                    }
-                }
-            }
-            this.lookup = lookup;
-        }
-
-
-        public void add(NumberWord number) {
-
-            if (numberClusters.getElements().contains(number)) {
-                return;
-            }
-
-            numberClusters.addElement(number);
-            for (NumberWord element : numberClusters.getElements()) {
-                if (matches(number, element, lookup)) {
-                    numberClusters.union(element, number);
-                }
-            }
-        }
-
-
-        public List<NumberWord> getCurrentRightmostCluster() {
-
-            return numberClusters.getGroups()
-                    .stream()
-                    .filter(cluster -> cluster.size() > MINIMUM_MATCHES)
-                    .map(cluster -> cluster.stream()
-                            .sorted(new TocNumberComparator(lookup))
-                            .toList())
-                    .map(this::removeOutliers)
-                    .map(this::removeOnNonConsecutivePages)
-                    .map(this::filterByWordNearTopOfPage)
-                    .filter(cluster -> cluster.size() > MINIMUM_MATCHES)
-                    .max(Comparator.comparingDouble(cluster -> cluster.get(0).word().getBBox().getMaxX())).orElse(Collections.emptyList());
-        }
-
-
-        private List<NumberWord> removeOnNonConsecutivePages(List<NumberWord> numbers) {
-
-            List<NumberWord> result = new ArrayList<>();
-
-            result.add(numbers.get(0));
-
-            for (int i = 1; i < numbers.size(); i++) {
-                int prev = getPageNumber(numbers, i - 1);
-                int curr = getPageNumber(numbers, i);
-
-                if (Math.abs(prev - curr) > 1) {
-                    break;
-                } else {
-                    result.add(numbers.get(i));
-                }
-            }
-            return result;
-        }
-
-
-        private int getPageNumber(List<NumberWord> numbers, int i) {
-
-            return lookup.get(numbers.get(i)).page().getPageNumber();
-        }
-
-
-        private List<NumberWord> filterByWordNearTopOfPage(List<NumberWord> numbers) {
-
-            List<NumberWord> result = new ArrayList<>();
-
-            result.add(numbers.get(0));
-
-            for (int i = 1; i < numbers.size(); i++) {
-                NumberWord prev = numbers.get(i - 1);
-                NumberWord curr = numbers.get(i);
-                ClassificationPage prevPage = lookup.get(prev).page();
-                ClassificationPage currPage = lookup.get(curr).page();
-                if (prevPage.equals(currPage)) {
-                    result.add(curr);
-                } else if (curr.word().getBBox().getMinY() < currPage.getPageHeight() * 0.33) {
-                    result.add(curr);
-                }
-            }
-            return result;
-        }
-
-
-        public List<NumberWord> removeOutliers(List<NumberWord> numbers) {
-
-            List<NumberWord> confirmedClusterNumbers = new ArrayList<>();
-
-            confirmedClusterNumbers.add(numbers.get(0));
-
-            for (int i = 1; i < numbers.size() - 1; i++) {
-                int prev = getNumberAsInt(numbers, i - 1);
-                int curr = getNumberAsInt(numbers, i);
-                int next = getNumberAsInt(numbers, i + 1);
-
-                if (!(curr <= prev || curr >= next) || !isBetterWithout(numbers, i)) {
-                    confirmedClusterNumbers.add(numbers.get(i));
-                }
-            }
-            if (getNumberAsInt(numbers, numbers.size() - 1) >= getLatestNumber(confirmedClusterNumbers)) {
-                confirmedClusterNumbers.add(numbers.get(numbers.size() - 1));
-            }
-
-            return confirmedClusterNumbers;
-        }
-
-
-        private static int getLatestNumber(List<NumberWord> confirmedClusterNumbers) {
-
-            return confirmedClusterNumbers.get(confirmedClusterNumbers.size() - 1).number();
-        }
-
-
-        // Helper method to check if removing the current number results in a better order
-        public static boolean isBetterWithout(List<NumberWord> numbers, int i) {
-
-            if (i == 0 || i == numbers.size() - 1) {
-                return false;
-            }
-
-            int prev = getNumberAsInt(numbers, i - 1);
-            int curr = getNumberAsInt(numbers, i);
-            int next = getNumberAsInt(numbers, i + 1);
-
-            return (prev <= next) && (Math.abs(prev - next) < Math.abs(prev - curr) + Math.abs(curr - next));
-        }
-
-    }
-
-
-    private static int getNumberAsInt(List<NumberWord> numbers, int i) {
-
-        return numbers.get(i).number();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java

@@ -84,7 +84,6 @@ public class DocumentGraphFactory {
                 .filter(SemanticNode::isLeaf)
                 .filter(node -> !node.getType().equals(NodeType.HEADER))
                 .filter(node -> !node.getType().equals(NodeType.FOOTER))
-                .filter(node -> !node.getType().equals(NodeType.IMAGE))
                 .map(SemanticNode::getTextBlock)
                 .map(TextBlock::getAtomicTextBlocks)
                 .flatMap(Collection::stream)
@@ -95,14 +94,14 @@ public class DocumentGraphFactory {
     private void addSections(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument, Context context, Document document) {
 
         for (TableOfContentItem tocItem : classificationDocument.getTableOfContents()) {
-            GenericSemanticNode parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
-            Optional<GenericSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
-                                                                                  parent,
-                                                                                  tocItem.getChildren().isEmpty(),
-                                                                                  tocItem.getNonEmptySectionBlocks(),
-                                                                                  tocItem.getImages(),
-                                                                                  context,
-                                                                                  document);
+            var parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
+            Optional<AbstractSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
+                                                                                   parent,
+                                                                                   tocItem.getChildren().isEmpty(),
+                                                                                   tocItem.getNonEmptySectionBlocks(),
+                                                                                   tocItem.getImages(),
+                                                                                   context,
+                                                                                   document);
             tocItem.setSection(section.orElse(null));
         }
     }
@@ -129,13 +128,13 @@ public class DocumentGraphFactory {
         textBlocks.add(originalTextBlock);
         textBlocks.addAll(textBlocksToMerge);
 
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSort(textBlocks), node, context, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock2(TextPositionOperations.mergeAndSort(textBlocks), node, context, page);
 
         if (node instanceof DuplicatedParagraph duplicatedParagraph) {
-            AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock(textBlocks.stream()
-                                                                                                      .flatMap(tb -> tb.getWords()
-                                                                                                              .stream())
-                                                                                                      .collect(Collectors.toList()), node, context, page);
+            AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock2(textBlocks.stream()
+                                                                                                       .flatMap(tb -> tb.getSequences()
+                                                                                                               .stream())
+                                                                                                       .collect(Collectors.toList()), node, context, page);
             duplicatedParagraph.setUnsortedLeafTextBlock(unsortedTextBlock);
         }
 
@@ -207,7 +206,7 @@ public class DocumentGraphFactory {
 
         Page page = context.getPage(textBlocks.get(0).getPage());
         Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.merge(textBlocks), footer, context, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock2(TextPositionOperations.merge(textBlocks), footer, context, page);
         List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer);
         footer.setTreeId(tocId);
         footer.setLeafTextBlock(textBlock);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SearchTextWithTextPositionFactory.java

@@ -12,7 +12,7 @@ import java.util.Objects;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
@@ -28,7 +28,7 @@ public class SearchTextWithTextPositionFactory {
     public static final double LINEBREAK_DELTA_TOLERANCE = 1.5;
 
 
-    public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<Word> sequences) {
+    public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<TextPositionSequence> sequences) {
 
         if (sequences.isEmpty() || sequences.stream()
                 .allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
@@ -40,7 +40,7 @@ public class SearchTextWithTextPositionFactory {
         RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
         RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").bBoxDirAdj(currentTextPosition.getBBoxDirAdj()).build();
 
-        for (Word word : sequences) {
+        for (TextPositionSequence word : sequences) {
             for (int i = 0; i < word.getTextPositions().size(); ++i) {
 
                 currentTextPosition = word.getTextPositions().get(i);
@@ -66,7 +66,7 @@ public class SearchTextWithTextPositionFactory {
         }
 
         List<Rectangle2D> positions = sequences.stream()
-                .map(Word::getTextPositions)
+                .map(TextPositionSequence::getTextPositions)
                 .flatMap(Collection::stream)
                 .map(RedTextPosition::getBBoxPdf)
                 .toList();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SectionNodeFactory.java

@@ -2,11 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;
 
 import static java.lang.String.format;
 import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.groupingBy;
 
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
@@ -15,6 +17,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
@@ -27,13 +30,13 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class SectionNodeFactory {
 
-    public Optional<GenericSemanticNode> addSection(LayoutParsingType layoutParsingType,
-                                                    GenericSemanticNode parentNode,
-                                                    boolean isLeaf,
-                                                    List<AbstractPageBlock> pageBlocks,
-                                                    List<ClassifiedImage> images,
-                                                    DocumentGraphFactory.Context context,
-                                                    Document document) {
+    public Optional<AbstractSemanticNode> addSection(LayoutParsingType layoutParsingType,
+                                                     GenericSemanticNode parentNode,
+                                                     boolean isLeaf,
+                                                     List<AbstractPageBlock> pageBlocks,
+                                                     List<ClassifiedImage> images,
+                                                     DocumentGraphFactory.Context context,
+                                                     Document document) {
 
         // This is for the case where we have images on a page without any text/footer/header.
         // The pageBlocks list is empty, but we still need to add those images to the document.
@@ -49,7 +52,8 @@ public class SectionNodeFactory {
         }
 
         AbstractSemanticNode section;
-        if (isLeaf) {
+        boolean containsTablesAndTextBlocks = containsTablesAndTextBlocks(pageBlocks);
+        if (isLeaf && !containsTablesAndTextBlocks) {
             section = Section.builder().documentTree(context.getDocumentTree()).build();
         } else {
             section = SuperSection.builder().documentTree(context.getDocumentTree()).build();
@@ -60,7 +64,6 @@ public class SectionNodeFactory {
         section.setTreeId(getTreeId(parentNode, context, section));
 
         addFirstHeadlineDirectlyToSection(layoutParsingType, pageBlocks, context, section, document);
-        boolean containsTablesAndTextBlocks = containsTablesAndTextBlocks(pageBlocks);
         if (containsTablesAndTextBlocks) {
             splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(layoutParsingType,
                                                                                                   section,
@@ -150,8 +153,7 @@ public class SectionNodeFactory {
     private boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {
 
         return pageBlocks.stream()
-                       .anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) //
-               && pageBlocks.stream()
+                       .anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
                        .anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
     }
 
@@ -234,4 +236,6 @@ public class SectionNodeFactory {
                 .toList();
     }
 
+
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/TableNodeFactory.java

@@ -4,6 +4,7 @@ import static java.util.Collections.emptyList;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@@ -11,12 +12,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
 import lombok.experimental.UtilityClass;
@@ -34,7 +36,10 @@ public class TableNodeFactory {
                          Document document) {
 
         setPageNumberInCells(tablesToMerge);
-
+        Set<Page> pages = tablesToMerge.stream()
+                .map(AbstractPageBlock::getPage)
+                .map(context::getPage)
+                .collect(Collectors.toSet());
         List<List<Cell>> mergedRows = tablesToMerge.stream()
                 .map(TablePageBlock::getRows)
                 .flatMap(Collection::stream)
@@ -115,7 +120,7 @@ public class TableNodeFactory {
         if (cell.getTextBlocks().isEmpty()) {
             tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page));
         } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getWords(), tableCell, context, page);
+            textBlock = context.getTextBlockFactory().buildAtomicTextBlock2(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
         } else if (firstTextBlockIsHeadline(cell)) {
             SectionNodeFactory.addSection(layoutParsingType,
@@ -129,8 +134,8 @@ public class TableNodeFactory {
                                           context,
                                           document);
         } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
-            List<Word> sequences = TextPositionOperations.mergeAndSort(cell.getTextBlocks());
-            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
+            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSort(cell.getTextBlocks());
+            textBlock = context.getTextBlockFactory().buildAtomicTextBlock2(sequences, tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
         } else {
             cell.getTextBlocks()

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/TextBlockFactory.java

@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;
 
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
@@ -17,14 +17,14 @@ public class TextBlockFactory {
     long textBlockIdx;
 
 
-    public AtomicTextBlock buildAtomicTextBlock(List<Word> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock2(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
 
         Integer numberOnPage = context.getAndIncrementTextBlockNumberOnPage(page);
         return buildAtomicTextBlock(sequences, parent, numberOnPage, page);
     }
 
 
-    public AtomicTextBlock buildAtomicTextBlock(List<Word> sequences, SemanticNode parent, Integer numberOnPage, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, Integer numberOnPage, Page page) {
 
         SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionDto(sequences);
         int offset = stringOffset;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/GraphicExtractorService.java

@@ -11,7 +11,7 @@ import org.springframework.stereotype.Service;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
 
 import lombok.RequiredArgsConstructor;
@@ -33,10 +33,10 @@ public class GraphicExtractorService {
                                                 PDPage pdPage,
                                                 int pageNumber,
                                                 CleanRulings cleanRulings,
-                                                List<Word> words,
+                                                List<TextPositionSequence> textPositionSequences,
                                                 boolean graphicsRaster) {
 
-        List<Box> characterBBoxes = getCharacterBBoxes(words);
+        List<Box> characterBBoxes = getCharacterBBoxes(textPositionSequences);
         List<Box> classifiedRulingsBoxes = getLineBBoxesOfAllClassifiedRulings(cleanRulings);
 
         GraphicBBDetector graphicBBDetector = new GraphicBBDetector(pdPage, true);
@@ -63,9 +63,9 @@ public class GraphicExtractorService {
     }
 
 
-    private List<Box> getCharacterBBoxes(List<Word> words) {
+    private List<Box> getCharacterBBoxes(List<TextPositionSequence> textPositionSequences) {
 
-        return words.stream()
+        return textPositionSequences.stream()
                 .map(BoundingBox::getBBoxPdf)
                 .map(Box::new)
                 .collect(Collectors.toList());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentGraphMapper.java

@@ -84,7 +84,6 @@ public class DocumentGraphMapper {
                 switch (entryData.getType()) {
                     case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
                     case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
-                    case IMAGE -> pages.forEach(page -> page.getImages().add((Image) node));
                     default -> textBlock.getAtomicTextBlocks()
                             .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
                 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/OutlineMapper.java

@@ -1,84 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
-
-import java.awt.geom.AffineTransform;
-import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Optional;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
-import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
-import com.knecon.fforesight.service.viewerdoc.model.Outline;
-
-import lombok.SneakyThrows;
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class OutlineMapper {
-
-    public Outline createOutline(Document document) {
-
-        Outline outline = new Outline();
-        addChildren(document, null, outline);
-        return outline;
-    }
-
-
-    public void addChildren(SemanticNode parentNode, Outline.Entry parentEntry, Outline outline) {
-
-        parentNode.streamChildren()
-                .filter(child -> child instanceof Section || child instanceof SuperSection)
-                .forEach(child -> {
-                    Optional<Headline> headline = findHeadline(child);
-                    if (headline.isPresent()) {
-                        Outline.Entry entry = buildEntry(child.getHeadline());
-                        if (parentEntry != null) {
-                            parentEntry.children().add(entry);
-                        } else {
-                            outline.getEntries().add(entry);
-                        }
-                        addChildren(child, entry, outline);
-                    } else {
-                        addChildren(child, parentEntry, outline);
-                    }
-                });
-    }
-
-
-    private static Optional<Headline> findHeadline(SemanticNode child) {
-
-        return child.streamChildren()
-                .filter(node -> node instanceof Headline)
-                .map(node -> (Headline) node)
-                .findFirst();
-    }
-
-
-    @SneakyThrows
-    private Outline.Entry buildEntry(Headline headline) {
-
-        Map<Page, Rectangle2D> bbox = headline.getBBox();
-        Rectangle2D r = bbox.get(headline.getFirstPage());
-        Point2D.Double position = new Point2D.Double(r.getMinX(), r.getMaxY());
-        PageInformation pageInformation = PageInformation.fromPage(headline.getFirstPage());
-
-        AffineTransform pdfToPage = CoordinateTransforms.calculateInitialUserSpaceCoordsToPageCoords(pageInformation);
-        pdfToPage.transform(position, position);
-
-        AffineTransform mirror = new AffineTransform(1, 0, 0, -1, 0, pageInformation.heightRot());
-        mirror.transform(position, position);
-
-        AffineTransform.getTranslateInstance(0, 5).transform(position, position);
-
-        Outline.JumpAction action = new Outline.JumpAction(headline.getFirstPage().getNumber(), position);
-        return new Outline.Entry(headline.getTextBlock().getSearchText(), action, new LinkedList<>());
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/LegacyPDFStreamEngine.java

@@ -289,7 +289,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
                                                  new int[]{code},
                                                  font,
                                                  fontSize,
-                                                 (int) (fontSize * textMatrix.getScalingFactorX() * textMatrix.getScalingFactorY())));
+                                                 (int) (fontSize * textMatrix.getScalingFactorX())));
             processTextPosition(new TextPosition(pageRotation,
                                                  pageSize.getWidth(),
                                                  pageSize.getHeight(),
@@ -303,7 +303,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
                                                  new int[]{code},
                                                  font,
                                                  fontSize,
-                                                 (int) (fontSize * textMatrix.getScalingFactorX() * textMatrix.getScalingFactorY())));
+                                                 (int) (fontSize * textMatrix.getScalingFactorX())));
         } else {
 
             processTextPosition(new TextPosition(pageRotation,
@@ -319,7 +319,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
                                                  new int[]{code},
                                                  font,
                                                  fontSize,
-                                                 (int) (fontSize * textMatrix.getScalingFactorX() * textMatrix.getScalingFactorY())));
+                                                 (int) (fontSize * textMatrix.getScalingFactorX())));
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/PDFLinesTextStripper.java

@@ -5,7 +5,6 @@ import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
@@ -40,7 +39,7 @@ import org.apache.pdfbox.text.TextPosition;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.Getter;
 import lombok.Setter;
@@ -51,8 +50,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 public class PDFLinesTextStripper extends PDFTextStripper {
 
-    private final static Set<String> DOT_LIKE_CHARACTERS = Set.of(".", "·", "•", "․", "‧", "∙", "⋅", "・", "．", "･", "…", "⸱", "﹒", "ꞏ");
-    private final List<Word> words = new ArrayList<>();
+    private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
     private final List<Ruling> rulings = new ArrayList<>();
     private final List<Ruling> graphicsPath = new ArrayList<>();
     @Setter
@@ -203,7 +201,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
 
         try {
             if (stroke && !getGraphicsState().getStrokingColor().isPattern() && isBlack(getGraphicsState().getStrokingColor()) || //
-                !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && isBlack(getGraphicsState().getNonStrokingColor())) {
+                    !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && isBlack(getGraphicsState().getNonStrokingColor())) {
                 rulings.addAll(path);
             }
         } catch (UnsupportedOperationException e) {
@@ -246,10 +244,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                 direction = textPositions.get(i).getDir();
             }
 
-            if (!words.isEmpty()) {
-                previous = words.get(words.size() - 1)
+            if (!textPositionSequences.isEmpty()) {
+                previous = textPositionSequences.get(textPositionSequences.size() - 1)
                         .getTextPositions()
-                        .get(words.get(words.size() - 1).getTextPositions().size() - 1);
+                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
             }
 
             if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
@@ -259,7 +257,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
 
             if (textPositions.get(i).getDir() != direction && startIndex != i) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 startIndex = i;
                 direction = textPositions.get(i).getDir();
             }
@@ -268,7 +266,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
             if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
                 if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
-                    words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 }
                 startIndex = i;
             }
@@ -276,97 +274,53 @@ public class PDFLinesTextStripper extends PDFTextStripper {
             if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
                 if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
-                    words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 }
                 startIndex = i;
             }
 
-            if (i > 0
-                && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))
-                && i <= textPositions.size() - 2) {
+            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i)
+                    .getUnicode()
+                    .equals("\t")) && i <= textPositions.size() - 2) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
                 if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
 
                     // Remove false sequence ends (whitespaces)
                     if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
                         for (TextPosition t : sublist) {
-                            words.get(words.size() - 1).add(t);
+                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                         }
                     } else {
-                        words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                        textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                     }
                 }
                 startIndex = i + 1;
             }
-            if (isDottedLineFollowedByWord(textPositions, i, startIndex)) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
-                startIndex = i;
-            }
-            if (isWordFollowedByDottedLine(textPositions, i, startIndex)) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i - 2);
-                words.add(new Word(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
-                startIndex = i - 2;
-            }
         }
 
         List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ")
-                                   || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0")
-                                   || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
+        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1)
+                .getUnicode()
+                .equals("\u00A0") || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
             sublist = sublist.subList(0, sublist.size() - 1);
         }
 
         if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
                 .getUnicode()
                 .equals("\t")))) {
-            if (previous != null
-                && sublist.get(0).getYDirAdj() == previous.getYDirAdj()
-                && sublist.get(0).getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
+            if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
+                    .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
                 for (TextPosition t : sublist) {
-                    words.get(words.size() - 1).add(t);
+                    textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                 }
             } else {
-                words.add(new Word(sublist, pageNumber, isParagraphStart));
+                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, isParagraphStart));
             }
         }
-
         super.writeString(text);
     }
 
 
-    private boolean isWordFollowedByDottedLine(List<TextPosition> textPositions, int i, int startIndex) {
-
-        return i - startIndex >= 4 //
-               && isDot(textPositions, i) //
-               && isDot(textPositions, i - 1) //
-               && isDot(textPositions, i - 2) //
-               && alphanumeric(textPositions, i - 3);
-    }
-
-
-    private static boolean isDottedLineFollowedByWord(List<TextPosition> textPositions, int i, int startIndex) {
-
-        return i - startIndex >= 4 //
-               && alphanumeric(textPositions, i)  //
-               && isDot(textPositions, i - 1) //
-               && isDot(textPositions, i - 2) //
-               && isDot(textPositions, i - 3);
-    }
-
-
-    private static boolean isDot(List<TextPosition> textPositions, int i) {
-
-        return DOT_LIKE_CHARACTERS.contains(textPositions.get(i).getUnicode());
-    }
-
-
-    private static boolean alphanumeric(List<TextPosition> textPositions, int i) {
-
-        return Character.isAlphabetic(textPositions.get(i).getUnicode().charAt(0)) || Character.isDigit(textPositions.get(i).getUnicode().charAt(0));
-    }
-
-
     public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {
 
         return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
@@ -383,16 +337,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
 
     public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {
 
-        return previous != null
-               && sublist.get(0).getYDirAdj() == previous.getYDirAdj()
-               && sublist.get(0).getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
+        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
+                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
     }
 
 
     @Override
     public String getText(PDDocument doc) throws IOException {
 
-        words.clear();
+        textPositionSequences.clear();
         rulings.clear();
         graphicsPath.clear();
         path_x = 0.0f;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/PDFTextStripper.java

@@ -25,22 +25,10 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.regex.Pattern;
 
+import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.pdfbox.cos.COSDictionary;
@@ -58,8 +46,6 @@ import org.apache.pdfbox.text.TextPositionComparator;
 
 import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;
 
-import lombok.Getter;
-
 /**
  * This is just a copy except i only adjusted lines 594-607 cause this is a bug in Pdfbox.
  * see S416.pdf
@@ -208,33 +194,40 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
     }
 
 
-    public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {
 
+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
+    {
         PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
-        if (this.currentMarkedContents.isEmpty()) {
+        if (this.currentMarkedContents.isEmpty())
+        {
             this.markedContents.add(markedContent);
-        } else {
-            PDMarkedContent currentMarkedContent = this.currentMarkedContents.peek();
-            if (currentMarkedContent != null) {
+        }
+        else
+        {
+            PDMarkedContent currentMarkedContent =
+                    this.currentMarkedContents.peek();
+            if (currentMarkedContent != null)
+            {
                 currentMarkedContent.addMarkedContent(markedContent);
             }
         }
         this.currentMarkedContents.push(markedContent);
     }
 
-
     @Override
-    public void endMarkedContentSequence() {
-
-        if (!this.currentMarkedContents.isEmpty()) {
+    public void endMarkedContentSequence()
+    {
+        if (!this.currentMarkedContents.isEmpty())
+        {
             this.currentMarkedContents.pop();
         }
     }
 
 
-    public void xobject(PDXObject xobject) {
-
-        if (!this.currentMarkedContents.isEmpty()) {
+    public void xobject(PDXObject xobject)
+    {
+        if (!this.currentMarkedContents.isEmpty())
+        {
             this.currentMarkedContents.peek().addXObject(xobject);
         }
     }
@@ -320,11 +313,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             endBookmarkPageNumber = -1;
         }
 
-        if (startBookmarkPageNumber == -1
-            && startBookmark != null
-            && endBookmarkPageNumber == -1
-            && endBookmark != null
-            && startBookmark.getCOSObject() == endBookmark.getCOSObject()) {
+        if (startBookmarkPageNumber == -1 && startBookmark != null && endBookmarkPageNumber == -1 && endBookmark != null && startBookmark.getCOSObject() == endBookmark.getCOSObject()) {
             // this is a special case where both the start and end bookmark
             // are the same but point to nothing. In this case
             // we will not extract any text.
@@ -371,9 +360,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
     @Override
     public void processPage(PDPage page) throws IOException {
 
-        if (currentPageNo >= startPage && currentPageNo <= endPage && (startBookmarkPageNumber == -1 || currentPageNo >= startBookmarkPageNumber) && (endBookmarkPageNumber == -1
-                                                                                                                                                      || currentPageNo
-                                                                                                                                                         <= endBookmarkPageNumber)) {
+        if (currentPageNo >= startPage && currentPageNo <= endPage && (startBookmarkPageNumber == -1 || currentPageNo >= startBookmarkPageNumber) && (endBookmarkPageNumber == -1 || currentPageNo <= endBookmarkPageNumber)) {
             startPage(page);
 
             int numberOfArticleSections = 1;
@@ -648,6 +635,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                         var normalized = normalize(line);
 //                        normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent()
 
+
                         lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine);
                         writeLine(normalized, current.isParagraphStart);
                         line.clear();
@@ -659,8 +647,8 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                     }
                     // test if our TextPosition starts after a new word would be expected to start
                     if (expectedStartOfNextWordX != EXPECTED_START_OF_NEXT_WORD_X_RESET_VALUE && expectedStartOfNextWordX < positionX
-                        // only bother adding a word separator if the last character was not a word separator
-                        && (wordSeparator.isEmpty() || //
+                            // only bother adding a word separator if the last character was not a word separator
+                            && (wordSeparator.isEmpty() || //
                             (lastPosition.getTextPosition().getUnicode() != null && !lastPosition.getTextPosition().getUnicode().endsWith(wordSeparator)))) {
                         line.add(LineItem.getWordSeparator());
                     }
@@ -926,7 +914,8 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                     textList.add(text);
                 }
             }
-            if (!this.currentMarkedContents.isEmpty()) {
+            if (!this.currentMarkedContents.isEmpty())
+            {
                 this.currentMarkedContents.peek().addText(text);
             }
         }
@@ -1722,6 +1711,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
         int numberOfStrings = line.size();
         for (int i = 0; i < numberOfStrings; i++) {
             WordWithTextPositions word = line.get(i);
+            word.getTextPositions().sort(Comparator.comparing(TextPosition::getXDirAdj));
             writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
             if (i < numberOfStrings - 1) {
                 writeWordSeparator();
@@ -2112,9 +2102,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             return endParagraphWritten;
         }
 
-
-        public void setEndParagraphWritten() {
-
+        public void setEndParagraphWritten(){
             endParagraphWritten = true;
         }
 
@@ -2157,6 +2145,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             this.isHangingIndent = true;
         }
 
+
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java

@@ -10,9 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.He
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.OutlineMapper;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutGrid;
-import com.knecon.fforesight.service.viewerdoc.model.Outline;
 import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;
 
 import io.micrometer.observation.annotation.Observed;
@@ -31,18 +29,16 @@ public class LayoutGridService {
 
     @SneakyThrows
     @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
-    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {
+    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue, boolean writeVisualLayoutParsingGrid) {
 
         LayoutGrid layoutGrid = createLayoutGrid(document);
-        Outline outline = OutlineMapper.createOutline(document);
         layoutGrid.setVisibleByDefault(layerVisibilityDefaultValue);
-
-        document.getLayoutDebugLayer().addSentenceVisualization(document.getTextBlock());
-
+//        Visualizations visualLayoutGrid = this.addLayoutGrid(document, layerVisibilityDefaultValue, true);
         if (document.getLayoutDebugLayer().isActive()) {
-            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid, document.getLayoutDebugLayer()), outline);
+            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid, document.getLayoutDebugLayer()));
         } else {
-            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid), outline);
+            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid));
+
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/HeaderFooterDetection.java

@@ -1,8 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;
 
-import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.FOOTER;
-import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.HEADER;
-
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -12,7 +9,6 @@ import java.util.stream.Collectors;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import lombok.experimental.UtilityClass;
@@ -30,60 +26,35 @@ public class HeaderFooterDetection {
 
     public boolean isLikelyFooter(TextPageBlock textPageBlock, ClassificationDocument document, ClassificationPage classificationPage) {
 
-        return isLikelyHeaderFooter(textPageBlock, document, classificationPage, FOOTER);
+        int numberOfPages = document.getPages().size();
+        if (numberOfPages < 3) {
+            // If the document has 1 or 2 pages this may lead to more false positives than actual findings.
+            return false;
+        }
+
+        int window = Math.min(numberOfPages, 8);
+
+        List<ClassificationPage> nearestPages = findNearestPages(classificationPage, document.getPages(), window);
+        List<List<AbstractPageBlock>> footerCandidates = getFooterCandidates(nearestPages);
+
+        return detectHeadersOrFootersByPageAssociation(textPageBlock.getText(), footerCandidates, window, footerWeights);
     }
 
 
     public boolean isLikelyHeader(TextPageBlock textPageBlock, ClassificationDocument document, ClassificationPage classificationPage) {
 
-        return isLikelyHeaderFooter(textPageBlock, document, classificationPage, HEADER);
-    }
-
-
-    private boolean isLikelyHeaderFooter(TextPageBlock textPageBlock, ClassificationDocument document, ClassificationPage classificationPage, PageBlockType type) {
-
         int numberOfPages = document.getPages().size();
         if (numberOfPages < 3) {
             // If the document has 1 or 2 pages this may lead to more false positives than actual findings.
             return false;
         }
 
-        List<TextPageBlock> textPageBlocks = classificationPage.getTextBlocks()
-                .stream()
-                .filter(TextPageBlock.class::isInstance)
-                .map(TextPageBlock.class::cast)
-                .collect(Collectors.toList());
-
-        if (textPageBlocks.isEmpty()) {
-            return false;
-        }
-
-        List<TextPageBlock> selectedBlocks;
-        if (type == HEADER) {
-            selectedBlocks = textPageBlocks.subList(0, Math.min(3, textPageBlocks.size()));
-        } else { //FOOTER
-            selectedBlocks = textPageBlocks.subList(Math.max(0, textPageBlocks.size() - 3), textPageBlocks.size());
-        }
-
-        if (!selectedBlocks.contains(textPageBlock)) {
-            // The textPageBlock is not among the selected blocks on its page
-            return false;
-        }
-
         int window = Math.min(numberOfPages, 8);
+
         List<ClassificationPage> nearestPages = findNearestPages(classificationPage, document.getPages(), window);
+        List<List<AbstractPageBlock>> headerCandidates = getHeaderCandidates(nearestPages);
 
-        List<List<AbstractPageBlock>> candidates;
-        double[] weights;
-        if (type == HEADER) {
-            candidates = getHeaderCandidates(nearestPages);
-            weights = headerWeights;
-        } else { //FOOTER
-            candidates = getFooterCandidates(nearestPages);
-            weights = footerWeights;
-        }
-
-        return detectHeadersOrFootersByPageAssociation(textPageBlock.getText(), candidates, window, weights);
+        return detectHeadersOrFootersByPageAssociation(textPageBlock.getText(), headerCandidates, window, headerWeights);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/MarkedContentUtils.java

@@ -13,7 +13,7 @@ import org.apache.pdfbox.text.TextPosition;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
@@ -48,7 +48,7 @@ public class MarkedContentUtils {
 
         return markedContentByYPosition.values()
                 .stream()
-                .map(textPositions -> new Word(textPositions, 0, true).getBBoxPdf())
+                .map(textPositions -> new TextPositionSequence(textPositions, 0, true).getBBoxPdf())
                 .map(t -> new Rectangle2D.Double(t.getX(), t.getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight())))
                 .collect(Collectors.toList());
     }
@@ -89,7 +89,7 @@ public class MarkedContentUtils {
                     .filter(content -> content instanceof TextPosition)
                     .map(content -> (TextPosition) content)
                     .filter(content -> !content.getUnicode().equals(" "))
-                    .map(textPositions -> new Word(List.of(textPositions), 0, true))
+                    .map(textPositions -> new TextPositionSequence(List.of(textPositions), 0, true))
                     .map(BoundingBox::getBBoxPdf)
                     .collect(Collectors.toList());
         }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/PageInformation.java

@@ -5,22 +5,14 @@ import java.awt.geom.Rectangle2D;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-
 public record PageInformation(Rectangle2D mediabox, int number, int rotationDegrees) {
 
     public static PageInformation fromPDPage(int pageNum, PDPage page) {
 
         PDRectangle mediaBox = page.getMediaBox();
         return new PageInformation(new Rectangle2D.Double(mediaBox.getLowerLeftX(), mediaBox.getLowerLeftY(), mediaBox.getWidth(), mediaBox.getHeight()),
-                                   pageNum,
-                                   page.getRotation());
-    }
-
-
-    public static PageInformation fromPage(Page page) {
-
-        return new PageInformation(new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight()), page.getNumber(), page.getRotation());
+                pageNum,
+                page.getRotation());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TableMergingUtility.java

@@ -36,7 +36,7 @@ public class TableMergingUtility {
             TablePageBlock consecutiveTable = consecutiveTables.get(i);
 
             if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() //
-                && getHeaders(consecutiveTable).isEmpty() //
+                && headersMatch(originalTablePageBlock, consecutiveTable) //
                 && outerBoundaryAlignsX(originalTablePageBlock, consecutiveTable) //
                 && consecutiveOrSamePage(currentTable, consecutiveTable) //
                 && !tableBetween(currentTable, consecutiveTable, findTablesBetween(consecutiveTables, currentTableIndex, i))) {
@@ -80,6 +80,12 @@ public class TableMergingUtility {
     }
 
 
+    private static boolean headersMatch(TablePageBlock originalTable, TablePageBlock consecutiveTable) {
+
+        return getHeaders(consecutiveTable).isEmpty() || getHeaders(originalTable).equals(getHeaders(consecutiveTable));
+    }
+
+
     private static boolean outerBoundaryAlignsX(TablePageBlock originalTablePageBlock, TablePageBlock consecutiveTable) {
 
         return Math.abs(consecutiveTable.getMinX() - originalTablePageBlock.getMinX()) < TABLE_ALIGNMENT_THRESHOLD

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextNormalizationUtilities.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;
 
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import lombok.experimental.UtilityClass;
@@ -10,7 +11,6 @@ public final class TextNormalizationUtilities {
     public static final Pattern hyphenLineBreaks = Pattern.compile("[-~‐‒⁻−﹣゠⁓‑\\u00AD][\\r\\n]+");
     public static final Pattern linebreaks = Pattern.compile("[\\r\\n]+");
     public static final Pattern doubleWhitespaces = Pattern.compile("\\s{2,}");
-    public static final Pattern WHITESPACE_REMOVAL = Pattern.compile("\\s+");
 
 
     public String cleanString(String value) {
@@ -37,11 +37,4 @@ public final class TextNormalizationUtilities {
 
         return linebreaks.matcher(value).replaceAll(" ");
     }
-
-
-    public String removeAllWhitespaces(String value) {
-
-        return WHITESPACE_REMOVAL.matcher(value).replaceAll("");
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionOperations.java

@@ -2,7 +2,6 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -16,7 +15,7 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Union
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
 
@@ -36,71 +35,65 @@ public class TextPositionOperations {
                     .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, X_THRESHOLD));
 
 
-    public List<Word> mergeAndSort(List<TextPageBlock> textBlocks) {
+    public List<TextPositionSequence> mergeAndSort(List<TextPageBlock> textBlocks) {
 
         var sequences = textBlocks.stream()
-                .flatMap(tb -> tb.getWords()
+                .flatMap(tb -> tb.getSequences()
                         .stream())
                 .collect(Collectors.toSet());
         return sortUsingLineDetection(sequences);
     }
 
 
-    public List<Word> sort(List<Word> sequences) {
+    public List<TextPositionSequence> sort(List<TextPositionSequence> sequences) {
 
         return sortUsingLineDetection(new HashSet<>(sequences));
     }
 
 
-    private List<Word> sortUsingLineDetection(Set<Word> sequences) {
+    private List<TextPositionSequence> sortUsingLineDetection(Set<TextPositionSequence> sequences) {
 
         return sortLines(groupByLine(sequences));
+
     }
 
 
-    public List<Word> sortLines(Collection<Set<Word>> lines) {
+    public List<TextPositionSequence> sortLines(Collection<Set<TextPositionSequence>> lines) {
 
-        List<List<Word>> lineBlocks = new ArrayList<>();
-        for (Set<Word> line : lines) {
-            List<Word> sortedLine = sortByXDirAdj(line);
-            if (!sortedLine.isEmpty()) {
-                lineBlocks.add(sortedLine);
-            }
-        }
-        // need to use old sorting, since COMPARATOR_DIR_ADJ is not transitive
-        QuickSort.sort(lineBlocks, Comparator.comparing(line -> line.get(0), COMPARATOR_DIR_ADJ));
-
-        List<Word> list = new ArrayList<>();
-        for (List<Word> words : lineBlocks) {
-            list.addAll(words);
-        }
-        return list;
-    }
-
-
-    private List<Word> sortByXDirAdj(Set<Word> line) {
-
-        return line.stream()
-                .sorted(Comparator.comparing(Word::getXDirAdj))
+        return lines.stream()
+                .map(TextPositionOperations::sortByXDirAdj)
+                .filter(line -> !line.isEmpty())
+                .sorted(Comparator.comparing(line -> line.get(0), COMPARATOR_DIR_ADJ))
+                .flatMap(Collection::stream)
                 .toList();
     }
 
 
-    public Collection<Set<Word>> groupByLine(Set<Word> sequences) {
+    private List<TextPositionSequence> sortByXDirAdj(Set<TextPositionSequence> line) {
+
+        return line.stream()
+                .sorted(Comparator.comparing(TextPositionSequence::getXDirAdj))
+                .toList();
+    }
+
+
+    public Collection<Set<TextPositionSequence>> groupByLine(Set<TextPositionSequence> sequences) {
 
         double maxLineDistance = sequences.stream()
-                                         .map(Word::getBBoxDirAdj)
-                                         .mapToDouble(RectangularShape::getHeight).average().orElse(10) * MAX_LINE_HEIGHT_FACTOR;
+                                         .map(TextPositionSequence::getBBoxDirAdj)
+                                         .mapToDouble(RectangularShape::getHeight).average()
+                                         .orElse(10) * MAX_LINE_HEIGHT_FACTOR;
         double maxXGap = sequences.stream()
-                                 .map(Word::getBBoxDirAdj)
-                                 .mapToDouble(RectangularShape::getWidth).average().orElse(75) * MAX_WORD_DISTANCE_FACTOR;
+                                 .map(TextPositionSequence::getBBoxDirAdj)
+                                 .mapToDouble(RectangularShape::getWidth).average()
+                                 .orElse(75) * MAX_WORD_DISTANCE_FACTOR;
 
-        UnionFind<Word> unionFind = new UnionFind<>(sequences);
+        UnionFind<TextPositionSequence> unionFind = new UnionFind<>(sequences);
 
-        for (Word sequence : sequences) {
-            for (Word sequence2 : sequences) {
+        for (TextPositionSequence sequence : sequences) {
+            for (TextPositionSequence sequence2 : sequences) {
 
-                if (sequence.equals(sequence2)) { // || unionFind.inSameSet(sequence, sequence2)) doing this is actually slower than not doing it
+                if (sequence.equals(sequence2) || unionFind.inSameSet(sequence, sequence2)) {
                     continue;
                 }
 
@@ -109,16 +102,11 @@ public class TextPositionOperations {
                 double normalizedVerticalDistance = Math.abs(sequence.getBBoxDirAdj().getCenterY() - sequence2.getBBoxDirAdj().getCenterY()) / maxLineDistance;
                 double normalizedHorizontalDistance = Math.abs(sequence.getBBoxDirAdj().getCenterX() - sequence2.getBBoxDirAdj().getCenterX()) / maxXGap;
 
-                if (sequence.getDir() != sequence2.getDir()) {
-                    continue;
-                }
-                if (Math.abs(sequence.getFontSize() - sequence2.getFontSize()) > 0.5 * Math.max(sequence.getFontSize(), sequence2.getFontSize())) {
-                    continue;
-                }
-                if (Math.pow(normalizedVerticalDistance, 2) + Math.pow(normalizedHorizontalDistance, 2) > 1) {
-                    continue;
-                }
-                if (!ANGLE_FILTER.matches(angle)) {
+                if (sequence.getDir() != sequence2.getDir()
+                    || Math.abs(sequence.getFontSize() - sequence2.getFontSize()) > 0.5 * Math.min(sequence.getFontSize(),
+                                                                                                   sequence2.getFontSize())
+                    || Math.pow(normalizedVerticalDistance, 2) + Math.pow(normalizedHorizontalDistance, 2) > 1
+                    || !ANGLE_FILTER.matches(angle)) {
                     continue;
                 }
 
@@ -144,10 +132,10 @@ public class TextPositionOperations {
     }
 
 
-    public List<Word> merge(List<TextPageBlock> textBlocks) {
+    public List<TextPositionSequence> merge(List<TextPageBlock> textBlocks) {
 
         return textBlocks.stream()
-                .map(TextPageBlock::getWords)
+                .map(TextPageBlock::getSequences)
                 .flatMap(Collection::stream)
                 .collect(Collectors.toList());
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/visualization/LayoutDebugLayer.java

@@ -5,11 +5,9 @@ import java.awt.geom.AffineTransform;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
-import java.text.BreakIterator;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -20,16 +18,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextBlockOnPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.NumberWord;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
@@ -60,14 +53,14 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
     Map<Integer, AtomicInteger> outlineObjectsWithoutPointsPerPage = new HashMap<>();
 
 
-    public void addTextVisualizations(List<Word> words, int pageNumber) {
+    public void addTextVisualizations(List<TextPositionSequence> textPositionSequences, int pageNumber) {
 
         if (!active) {
             return;
         }
         VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(pageNumber, this.words);
         visualizationsOnPage.getColoredRectangles()
-                .addAll(words.stream()
+                .addAll(textPositionSequences.stream()
                                 .map(BoundingBox::getBBoxPdf)
                                 .map(rect -> new ColoredRectangle(rect, WORDS_COLOR, 1))
                                 .toList());
@@ -83,7 +76,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         visualizationsOnPage.getColoredLines()
                 .addAll(cleanRulings.buildAll()
                                 .stream()
-                                .map(ruling -> new ColoredLine(ruling, decideOnRulingColor(ruling), LINE_WIDTH))
+                                .map(ruling -> new ColoredLine(ruling, decideOnRulingColor(ruling), 0.5f))
                                 .toList());
     }
 
@@ -96,34 +89,11 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(pageNumber, this.rulings);
         visualizationsOnPage.getColoredLines()
                 .addAll(rulings.stream()
-                                .map(ruling -> new ColoredLine(ruling, decideOnRulingColor(ruling), LINE_WIDTH))
+                                .map(ruling -> new ColoredLine(ruling, decideOnRulingColor(ruling), 0.5f))
                                 .toList());
     }
 
 
-    public void addSentenceVisualization(TextBlock textBlock) {
-
-        if (!active) {
-            return;
-        }
-        AtomicInteger rotatingColorIdx = new AtomicInteger(0);
-        String text = textBlock.getSearchText();
-        BreakIterator sentenceIterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
-        sentenceIterator.setText(text);
-        int lastIdx = 0;
-        while (sentenceIterator.next() != BreakIterator.DONE) {
-            TextRange sentenceRange = new TextRange(lastIdx + textBlock.getTextRange().start(), sentenceIterator.current() + textBlock.getTextRange().start());
-            lastIdx = sentenceIterator.current();
-            Color color = getRotatingColor(rotatingColorIdx);
-            textBlock.getPositionsPerPage(sentenceRange)
-                    .forEach((page, bboxes) -> getOrCreateVisualizationsOnPage(page.getNumber(), this.sentences).getColoredRectangles()
-                            .addAll(bboxes.stream()
-                                            .map(bbox -> new ColoredRectangle(bbox, color, 1))
-                                            .toList()));
-        }
-    }
-
-
     private Color decideOnRulingColor(Ruling ruling) {
 
         return switch (ruling.getClassification()) {
@@ -185,12 +155,12 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         visualizationsOnPage.getColoredRectangles()
                 .addAll(lines.stream()
                                 .map(BoundingBox::getBBoxPdf)
-                                .map(line -> new ColoredRectangle(line, LINES_COLOR, LINE_WIDTH))
+                                .map(line -> new ColoredRectangle(line, LINES_COLOR, 0.5f))
                                 .toList());
     }
 
 
-    public void addLineVisualizationsFromNestedTextPosition(Collection<Set<Word>> lines, int pageNumber) {
+    public void addLineVisualizationsFromNestedTextPosition(Collection<Set<TextPositionSequence>> lines, int pageNumber) {
 
         if (!active) {
             return;
@@ -201,13 +171,13 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
                                 .map(line -> line.stream()
                                         .map(BoundingBox::getBBoxPdf)
                                         .collect(RectangleTransformations.collectBBox()))
-                                .map(line -> new ColoredRectangle(line, LINES_COLOR, LINE_WIDTH))
+                                .map(line -> new ColoredRectangle(line, LINES_COLOR, 0.5f))
                                 .toList());
 
     }
 
 
-    public void addTextBlockVisualizations(List<AbstractPageBlock> textPageBlocks, int page) {
+    public void addTextBlockVisualizations(List<TextPageBlock> textPageBlocks, int page) {
 
         if (!active) {
             return;
@@ -278,7 +248,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
                 .map(Line::getCharacters)
                 .flatMap(Collection::stream)
                 .forEach(character -> {
-                    Color color = getRotatingColor(index);
+                    Color color = ROTATING_CHARACTER_COLOR.get(index.getAndIncrement() % ROTATING_CHARACTER_COLOR.size());
                     Rectangle2D charBBox = character.getTextPosition().getBBoxPdf();
                     characterVisualizations.getColoredRectangles().add(new ColoredRectangle(charBBox, color, 1));
                     character.getNeighbors()
@@ -293,28 +263,6 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
     }
 
 
-    public void addTocPages(List<NumberWord> numbers, int page) {
-
-        if (!active) {
-            return;
-        }
-
-        VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(page, this.tocPages);
-        visualizationsOnPage.getColoredRectangles()
-                .addAll(numbers.stream()
-                                .map(NumberWord::word)
-                                .map(BoundingBox::getBBoxPdf)
-                                .map(line -> new ColoredRectangle(line, LINES_COLOR, LINE_WIDTH))
-                                .toList());
-    }
-
-
-    private static Color getRotatingColor(AtomicInteger index) {
-
-        return ROTATING_CHARACTER_COLOR.get(index.getAndIncrement() % ROTATING_CHARACTER_COLOR.size());
-    }
-
-
     public void addOutlineObjects(List<OutlineObject> outlineObjects, PageInformation pageInformation) {
 
         if (!active) {
@@ -329,10 +277,8 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
 
     private void addOutlineObject(OutlineObject outlineObject, PageInformation pageInformation) {
 
-        if (!active) {
-            return;
-        }
         int rectSize = 5;
+
         Point2D point2D;
         if (outlineObject.getPoint().isPresent()) {
             point2D = outlineObject.getPoint().get();
@@ -353,28 +299,4 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         visualizationsOnPage.getPlacedTexts().add(PlacedText.textFacingUp(outlineObject.getTitle(), textPoint, 10, outlineObject.isFound() ? Color.BLACK : Color.RED, FONT));
     }
 
-
-    public void addListIdentifiers(List<ListIdentifier> listIdentifiers) {
-
-        if (!active) {
-            return;
-        }
-        for (ListIdentifier listIdentifier : listIdentifiers) {
-            getOrCreateVisualizationsOnPage(listIdentifier.getPage(), this.listIdentifiers).getColoredRectangles()
-                    .add(new ColoredRectangle(listIdentifier.getWord().getBBoxPdf(), WORDS_COLOR, LINE_WIDTH));
-        }
-    }
-
-
-    public void addTocBlocks(Set<TextBlockOnPage> blocksWithNumberInCluster) {
-
-        if (!active) {
-            return;
-        }
-        for (TextBlockOnPage textBlockOnPage : blocksWithNumberInCluster) {
-            getOrCreateVisualizationsOnPage(textBlockOnPage.page().getPageNumber(), this.tocBlocks).getColoredRectangles()
-                    .add(new ColoredRectangle(textBlockOnPage.textBlock().getBBoxPdf(), TOC_COLOR, LINE_WIDTH));
-        }
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/visualization/LayoutGrid.java

@@ -88,19 +88,7 @@ public class LayoutGrid extends LayoutGridLayerConfig {
     public void addTreeId(SemanticNode semanticNode) {
 
         Page page = semanticNode.getFirstPage();
-        if (semanticNode.getBBox()
-                    .get(page) == null) {
-            return;
-        }
-        addPlacedText(page,
-                      semanticNode.getBBox()
-                              .get(page),
-                      semanticNode.getBBox()
-                              .get(page),
-                      buildTreeIdString(semanticNode),
-                      1,
-                      treeIds,
-                      TREEID_COLOR);
+        addPlacedText(page, semanticNode.getBBox().get(page), semanticNode.getBBox().get(page), buildTreeIdString(semanticNode), 1, treeIds, TREEID_COLOR);
     }
 
 
@@ -129,8 +117,7 @@ public class LayoutGrid extends LayoutGridLayerConfig {
                 .toList();
         Integer maxChildDepth = subSections.stream()
                 .map(node -> node.getTreeId().size())
-                .max(Integer::compareTo)
-                .orElse(section.getTreeId().size());
+                .max(Integer::compareTo).orElse(section.getTreeId().size());
         int ownDepth = section.getTreeId().size();
 
         Page firstPage = section.getFirstPage();
@@ -146,6 +133,9 @@ public class LayoutGrid extends LayoutGridLayerConfig {
                 .collect(Collectors.toList());
         pagesInOrder.remove(0);
         handleFirstPageOfSection(section, firstPage, bBoxMap.get(firstPage), treeIdString, maxChildDepth, ownDepth);
+        if (section instanceof SuperSection) {
+            return;
+        }
         for (Page middlePage : pagesInOrder.subList(0, pagesInOrder.size() - 1)) {
             handleForMiddlePageOfSection(section, middlePage, bBoxMap.get(middlePage), treeIdString, maxChildDepth, ownDepth);
         }
@@ -213,10 +203,9 @@ public class LayoutGrid extends LayoutGridLayerConfig {
         List<PlacedText> placedTexts = getOrCreateVisualizationsOnPage(page.getNumber(), visualizations).getPlacedTexts();
 
         PlacedText newText = PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, color, FONT);
-        float threshold = 1.5f * FONT_SIZE;
+
         Optional<PlacedText> conflictingText = placedTexts.stream()
-                .filter(pt -> Math.abs(pt.lineStart().getY() - newText.lineStart().getY()) <= threshold
-                              && Math.abs(pt.lineStart().getX() - newText.lineStart().getX()) <= threshold)
+                .filter(pt -> Math.abs(pt.lineStart().getY() - newText.lineStart().getY()) <= FONT_SIZE)
                 .findFirst();
 
         if (conflictingText.isPresent()) {
@@ -297,8 +286,7 @@ public class LayoutGrid extends LayoutGridLayerConfig {
 
         List<ColoredLine> coloredLines = getOrCreateVisualizationsOnPage(page.getNumber(), sections).getColoredLines();
         int lineWidthModifier = maxChildDepth - ownDepth;
-        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox()
-                                                             .get(page), LINE_WIDTH * (1 + lineWidthModifier), LINE_WIDTH * (1 + lineWidthModifier));
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(page), LINE_WIDTH * (1 + lineWidthModifier), LINE_WIDTH * (1 + lineWidthModifier));
 
         SemanticNode highestParent = semanticNode.getHighestParent();
         Rectangle2D highestParentRect = rectangleMap.get(new RectangleIdentifier(highestParent.getTreeId(), page.getNumber()));
@@ -347,8 +335,7 @@ public class LayoutGrid extends LayoutGridLayerConfig {
             List<Double> ys = yStream.collect(Collectors.toList());
             ys.remove(0);
 
-            Rectangle2D tableBBox = table.getBBox()
-                    .get(page);
+            Rectangle2D tableBBox = table.getBBox().get(page);
             List<ColoredLine> coloredLines = getOrCreateVisualizationsOnPage(page.getNumber(), tables).getColoredLines();
 
             xs.forEach(x -> {

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -30,7 +30,7 @@ dependencies {
     implementation(project(":layoutparser-service-internal-api"))
 
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.29.0")
     implementation("com.knecon.fforesight:tracing-commons:0.5.0")
     implementation("com.knecon.fforesight:lifecycle-commons:0.6.0")
     implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/configuration/MessagingConfiguration.java

@@ -0,0 +1,29 @@
+package com.knecon.fforesight.service.layoutparser.server.configuration;
+
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
+
+import org.springframework.amqp.core.DirectExchange;
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+
+@Configuration
+public class MessagingConfiguration {
+
+    @Bean
+    public DirectExchange layoutParsingRequestExchange() {
+
+        return new DirectExchange(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue layoutParsingDLQ() {
+
+        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/configuration/TenantMessagingConfigurationImpl.java

@@ -0,0 +1,11 @@
+package com.knecon.fforesight.service.layoutparser.server.configuration;
+
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
+
+@Configuration
+public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
+
+
+}

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -18,6 +18,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -28,15 +29,17 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class MessageHandler {
 
+    public static final String LAYOUT_PARSING_REQUEST_LISTENER_ID = "layout-parsing-request-listener";
+
     private final LayoutParsingPipeline layoutParsingPipeline;
     private final ObjectMapper objectMapper;
     private final RabbitTemplate rabbitTemplate;
     private final static String X_PIPELINE_PREFIX = "X-PIPE-";
 
 
-    @RabbitHandler
-    @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)
     @SneakyThrows
+    @RabbitHandler
+    @RabbitListener(id = LAYOUT_PARSING_REQUEST_LISTENER_ID)
     public void receiveLayoutParsingRequest(Message message) {
 
         LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);
@@ -57,7 +60,7 @@ public class MessageHandler {
     public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent, Message message) {
 
         Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent, m -> {
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE, TenantContext.getTenantId(), layoutParsingFinishedEvent, m -> {
             var forwardHeaders = message.getMessageProperties()
                     .getHeaders()
                     .entrySet()

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/TenantExchangeMessageReceiverImpl.java

@@ -0,0 +1,64 @@
+package com.knecon.fforesight.service.layoutparser.server.queue;
+
+import java.util.Set;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+import com.knecon.fforesight.tenantcommons.TenantProvider;
+import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
+import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
+import com.knecon.fforesight.tenantcommons.model.TenantResponse;
+import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
+import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
+
+@Service
+public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
+
+    public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
+
+        super(rabbitQueueService, tenantProvider);
+    }
+
+
+    @Override
+    protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
+
+        return Set.of(TenantQueueConfiguration.builder()
+                              .listenerId(MessageHandler.LAYOUT_PARSING_REQUEST_LISTENER_ID)
+                              .exchangeName(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE)
+                              .queuePrefix(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE_PREFIX)
+                              .dlqName(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ)
+                              .build());
+    }
+
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void onApplicationReady() {
+
+        System.out.println("application ready invoked");
+        super.initializeQueues();
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
+    public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
+
+        super.reactToTenantCreation(tenantCreatedEvent);
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
+    public void reactToTenantDeletion(TenantResponse tenantResponse) {
+
+        super.reactToTenantDeletion(tenantResponse);
+
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/main/resources/logback-spring.xml

@@ -14,6 +14,4 @@
         <appender-ref ref="${logType}"/>
     </root>
 
-    <logger name="org.apache.fontbox.ttf" level="ERROR"/>
-
 </configuration>

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/BoundingBoxTest.java

@@ -1,70 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.awt.geom.Rectangle2D;
-
-import org.junit.jupiter.api.Test;
-
-class BoundingBoxTest {
-
-    @Test
-    void testHorizontalDistance_NoOverlap() {
-
-        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
-        ConcreteBoundingBox box2 = new ConcreteBoundingBox(20, 0, 10, 10);
-
-        assertEquals(10, box1.horizontalDistance(box2));
-    }
-
-
-    @Test
-    void testHorizontalDistance_Overlap() {
-
-        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
-        ConcreteBoundingBox box2 = new ConcreteBoundingBox(5, 0, 10, 10);
-
-        assertEquals(0, box1.horizontalDistance(box2));
-    }
-
-
-    @Test
-    void testVerticalDistance_NoOverlap() {
-
-        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
-        ConcreteBoundingBox box2 = new ConcreteBoundingBox(0, 20, 10, 10);
-
-        assertEquals(10, box1.verticalDistance(box2));
-    }
-
-
-    @Test
-    void testVerticalDistance_Overlap() {
-
-        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
-        ConcreteBoundingBox box2 = new ConcreteBoundingBox(0, 5, 10, 10);
-
-        assertEquals(0, box1.verticalDistance(box2));
-    }
-
-
-    @Test
-    void testVerticalDistance_PartialOverlap() {
-
-        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
-        ConcreteBoundingBox box2 = new ConcreteBoundingBox(0, 8, 10, 10);
-
-        assertEquals(0, box1.verticalDistance(box2));
-    }
-
-
-    @Test
-    void testHorizontalDistance_PartialOverlap() {
-
-        ConcreteBoundingBox box1 = new ConcreteBoundingBox(0, 0, 10, 10);
-        ConcreteBoundingBox box2 = new ConcreteBoundingBox(8, 0, 10, 10);
-
-        assertEquals(0, box1.horizontalDistance(box2));
-    }
-
-}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/ConcreteBoundingBox.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
-
-import java.awt.geom.Rectangle2D;
-
-class ConcreteBoundingBox extends BoundingBox {
-
-    ConcreteBoundingBox(double x, double y, double width, double height) {
-
-        this.bBox = new Rectangle2D.Double(x, y, width, height);
-    }
-
-}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/processor/model/SectionIdentifierTest.java

@@ -1,93 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.util.List;
-
-import org.junit.jupiter.api.Test;
-
-class SectionIdentifierTest {
-
-    @Test
-    void testSectionIdentifier() {
-
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText("1.1.2: Headline");
-        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
-        assertEquals(3, identifier.level());
-        assertEquals(List.of(1, 1, 2), identifier.getIdentifiers());
-
-        SectionIdentifier child = SectionIdentifier.asChildOf(identifier);
-        assertTrue(child.isChildOf(identifier));
-
-        SectionIdentifier parent = SectionIdentifier.fromSearchText("1.1: Headline");
-        assertTrue(parent.isParentOf(identifier));
-    }
-
-
-    @Test
-    void testSectionIdentifier2() {
-
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText("A.1.2: Headline");
-        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
-        assertEquals(3, identifier.level());
-        assertEquals(List.of(1, 1, 2), identifier.getIdentifiers());
-    }
-
-
-    @Test
-    void testSectionIdentifier3() {
-
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText("D.1.2: Headline");
-        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
-        assertEquals(3, identifier.level());
-        assertEquals(List.of(4, 1, 2), identifier.getIdentifiers());
-    }
-
-
-    @Test
-    void testSectionIdentifier4() {
-
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText("4.1.2.4: Headline");
-        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
-        assertEquals(4, identifier.level());
-        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
-    }
-
-
-    @Test
-    void testSectionIdentifier5() {
-
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText("D.1.2.4.5: Headline");
-        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
-        assertEquals(4, identifier.level());
-        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
-    }
-
-
-    @Test
-    void testSectionIdentifier6() {
-
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText("d.1.2.4.5: Headline");
-        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
-        assertEquals(4, identifier.level());
-        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
-    }
-
-
-    @Test
-    void testSectionIdentifier7() {
-
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText("4.1.2.4.5: Headline");
-        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
-        assertEquals(4, identifier.level());
-        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
-    }
-
-    @Test
-    void testFalsePositive111() {
-        SectionIdentifier identifier = SectionIdentifier.fromSearchText("111: Headline");
-        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
-        assertEquals(1, identifier.level());
-    }
-
-}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -10,7 +10,9 @@ import java.util.stream.Collectors;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -55,6 +57,12 @@ public class HeadlinesGoldStandardIntegrationTest {
     @MockBean
     private RabbitTemplate rabbitTemplate;
 
+    @MockBean
+    private RabbitAdmin rabbitAdmin;
+
+    @MockBean
+    private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
+
     @Autowired
     private ObjectMapper objectMapper;
 
@@ -77,9 +85,11 @@ public class HeadlinesGoldStandardIntegrationTest {
                                "files/headlineTest/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
 
         double precision = metrics.stream()
-                .mapToDouble(Metrics::getPrecision).average().orElse(1.0);
+                .mapToDouble(Metrics::getPrecision).average()
+                .orElse(1.0);
         double recall = metrics.stream()
-                .mapToDouble(Metrics::getRecall).average().orElse(1.0);
+                .mapToDouble(Metrics::getRecall).average()
+                .orElse(1.0);
 
         System.out.println("Precision is: " + precision + " recall is: " + recall);
 
@@ -96,9 +106,12 @@ public class HeadlinesGoldStandardIntegrationTest {
 
         Set<Headline> goldStandardHeadlines = new HashSet<>();
         var goldStandardLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class);
-        goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
         goldStandardLog.getRedactionLogEntry()
-                .forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));
+                .removeIf(r -> !r.isRedacted() || r.getChanges()
+                        .get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
+        goldStandardLog.getRedactionLogEntry()
+                .forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions()
+                                                                             .get(0).getPage(), e.getValue())));
 
         Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE,
                                                                          layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
@@ -113,7 +126,8 @@ public class HeadlinesGoldStandardIntegrationTest {
                 .distinct()
                 .map(headlineNode -> new Headline(headlineNode.getPages()
                                                           .stream()
-                                                          .findFirst().get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing()))
+                                                          .findFirst()
+                                                          .get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing()))
                 .toList();
 
         Set<Headline> correct = new HashSet<>();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java

@@ -37,7 +37,7 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
     @Disabled
     public void testLayoutParserEndToEnd() {
 
-        String filePath = "/home/kschuettler/Dokumente/LayoutparsingEvaluation/VV-340050.pdf";
+        String filePath = "/home/kschuettler/Dokumente/Ticket Related/RED-9964/17a25133-e098-4610-b553-d1bf11a56d96/560e6ab1ab4754b9a62fd2e6d4d71327/560e6ab1ab4754b9a62fd2e6d4d71327.ORIGIN.pdf";
 
         runForFile(filePath);
     }
@@ -48,7 +48,7 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
     @SneakyThrows
     public void testLayoutParserEndToEndWithFolder() {
 
-        String folder = "/home/kschuettler/Dokumente/Ticket Related/RED-9975";
+        String folder = "/home/kschuettler/Dokumente/TestFiles/ReadingOrder";
         List<Path> pdfFiles = Files.walk(Path.of(folder))
                 .filter(path -> path.getFileName().toString().endsWith(".pdf"))
                 .sorted(Comparator.comparing(Path::getFileName))

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/OutlineDetectionTest.java

@@ -36,6 +36,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.visualizati
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;
 
+import jakarta.annotation.PostConstruct;
 import lombok.SneakyThrows;
 
 public class OutlineDetectionTest extends AbstractTest {
@@ -79,9 +80,8 @@ public class OutlineDetectionTest extends AbstractTest {
         var documentFile = new ClassPathResource(fileName).getFile();
 
         long start = System.currentTimeMillis();
-        ClassificationDocument classificationDocument = parseLayout(fileName, LayoutParsingType.DOCUMINE_OLD);
-        Document document = buildGraph(fileName, classificationDocument);
-        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
+        ClassificationDocument classificationDocument = parseLayout(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
+
         OutlineObjectTree outlineObjectTree = classificationDocument.getOutlineObjectTree();
         assertEquals(outlineObjectTree.getRootNodes().size(), 8);
         assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(2).size(), 1);
@@ -111,15 +111,17 @@ public class OutlineDetectionTest extends AbstractTest {
                              .stream()
                              .map(outlineObjectTreeNode -> sanitizeString(outlineObjectTreeNode.getOutlineObject().getTitle()))
                              .toList());
-//        assertEquals(tableOfContents.getMainSections().get(5).getChildren().size(), 6);
-//        assertEquals(tableOfContents.getMainSections().get(7).getChildren().size(), 3);
-//        assertEquals(tableOfContents.getMainSections().get(8).getChildren().size(), 3);
-//        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().size(), 1);
-//        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().get(0).getChildren().size(), 3);
-//
-//        assertEquals(tableOfContents.getMainSections().get(0).getImages().size(), 1);
-//        assertEquals(tableOfContents.getMainSections().get(6).getImages().size(), 1);
-//        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().get(0).getChildren().get(2).getImages().size(), 1);
+        assertEquals(tableOfContents.getMainSections().get(5).getChildren().size(), 6);
+        assertEquals(tableOfContents.getMainSections().get(7).getChildren().size(), 3);
+        assertEquals(tableOfContents.getMainSections().get(8).getChildren().size(), 3);
+        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().size(), 1);
+        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().get(0).getChildren().size(), 3);
+
+        assertEquals(tableOfContents.getMainSections().get(0).getImages().size(), 1);
+        assertEquals(tableOfContents.getMainSections().get(6).getImages().size(), 1);
+        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().get(0).getChildren().get(2).getImages().size(), 1);
+
+        Document document = buildGraph(fileName, classificationDocument);
 
         assertTrue(tableOfContents.getAllTableOfContentItems()
                            .stream()
@@ -144,37 +146,38 @@ public class OutlineDetectionTest extends AbstractTest {
                              .stream()
                              .map(outlineObjectTreeNode -> sanitizeString(outlineObjectTreeNode.getOutlineObject().getTitle()))
                              .toList());
-//        Predicate<SemanticNode> isSectionOrSuperSection = semanticNode -> semanticNode instanceof Section || semanticNode instanceof SuperSection;
-//        assertEquals(childrenOfTypeSectionOrSuperSection.get(5).streamChildren()
-//                             .filter(isSectionOrSuperSection)
-//                             .count(), 6 + 1); // 1 additional for main text of parent section
-//        assertEquals(childrenOfTypeSectionOrSuperSection.get(7).streamChildren()
-//                             .filter(isSectionOrSuperSection)
-//                             .count(), 3 + 1);
-//        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
-//                             .filter(isSectionOrSuperSection)
-//                             .count(), 3 + 1);
-//        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
-//                             .filter(isSectionOrSuperSection)
-//                             .toList().get(3).streamChildren()
-//                             .filter(isSectionOrSuperSection)
-//                             .count(), 1 + 1);
-//        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
-//                             .filter(isSectionOrSuperSection)
-//                             .toList().get(3).streamChildren()
-//                             .filter(isSectionOrSuperSection)
-//                             .toList().get(1).streamChildren()
-//                             .filter(isSectionOrSuperSection)
-//                             .count(), 3 + 1);
+        Predicate<SemanticNode> isSectionOrSuperSection = semanticNode -> semanticNode instanceof Section || semanticNode instanceof SuperSection;
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(5).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 6 + 1); // 1 additional for main text of parent section
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(7).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList().get(3).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 1 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList().get(3).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList().get(1).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
 
-//        List<List<Integer>> imageTreeIdList = document.streamAllImages()
-//                .map(image -> image.getParent().getTreeId())
-//                .toList();
-//
-//        assertEquals(imageTreeIdList.get(0), List.of(0));
-//        assertEquals(imageTreeIdList.get(1), List.of(6));
-//        assertEquals(imageTreeIdList.get(2), List.of(8, 4, 2, 4));
+        List<List<Integer>> imageTreeIdList = document.streamAllImages()
+                .map(image -> image.getParent().getTreeId())
+                .toList();
 
+        assertEquals(imageTreeIdList.get(0), List.of(0));
+        assertEquals(imageTreeIdList.get(1), List.of(6));
+        assertEquals(imageTreeIdList.get(2), List.of(8, 4, 2, 4));
+
+        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/SimplifiedTextServiceTest.java

@@ -46,8 +46,8 @@ public class SimplifiedTextServiceTest
         Document document = buildGraph(file);
         SimplifiedText simplifiedText = simplifiedSectionTextService.toSimplifiedText(document);
         List<SimplifiedSectionText> sectionTexts = simplifiedText.getSectionTexts();
-        assertThat(sectionTexts.stream().filter(section -> section.getText().contains(footerExample)).toList().size()).isGreaterThan(0);
-        assertThat(sectionTexts.stream().filter(section -> section.getText().contains(headerExample)).toList().size()).isGreaterThan(0);
+        assertThat(sectionTexts.stream().filter(section -> section.getText().equals(footerExample)).collect(Collectors.toList()).size()).isGreaterThan(0);
+        assertThat(sectionTexts.stream().filter(section -> section.getText().equals(headerExample)).collect(Collectors.toList()).size()).isGreaterThan(0);
 
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentReadingOrderTest.java

@@ -13,7 +13,6 @@ import java.util.List;
 import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -26,7 +25,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 import com.knecon.fforesight.service.layoutparser.server.PDFNetInitializer;
@@ -78,20 +77,6 @@ public class DocumentReadingOrderTest extends BuildDocumentTest {
     }
 
 
-    @Test
-    @Disabled
-    public void drawDirAdjForFile() {
-
-        String pdfFile = "/home/kschuettler/Dokumente/TestFiles/OCR/VV-331340/VV-331340_OCRED_first15.pdf";
-
-        ClassificationDocument classificationDocument = parseLayout(pdfFile, LayoutParsingType.DOCUMINE_OLD);
-
-        drawDirAdjCoords(pdfFile, classificationDocument, LayoutParsingType.DOCUMINE_OLD);
-    }
-
-
-    @Disabled // Does not pass because now 27 and Document 10350420.doc Certificate of Analysis
-    // Page 1 of 1 Study T000973-08 is now header and footer // TODO check this again
     @Test
     public void readingOrderTestSeite14() {
 
@@ -387,7 +372,7 @@ public class DocumentReadingOrderTest extends BuildDocumentTest {
                 for (AbstractPageBlock abstractBlock : classificationDocumentPage.getTextBlocks()) {
 
                     if (abstractBlock instanceof TextPageBlock textBlock) {
-                        for (Word sequence : TextPositionOperations.mergeAndSort(List.of(textBlock))) {
+                        for (TextPositionSequence sequence : TextPositionOperations.mergeAndSort(List.of(textBlock))) {
 
                             float stringWidth;
                             try {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/TextRangeTest.java

@@ -76,10 +76,9 @@ class TextRangeTest {
         assertEquals(List.of(new TextRange(10, 40), new TextRange(40, 100)), startTextRange.split(List.of(40)));
         assertEquals(1, startTextRange.split(Collections.emptyList()).size());
         assertEquals(1, startTextRange.split(List.of(startTextRange.start())).size());
-        assertEquals(1, startTextRange.split(List.of(100)).size());
         assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(Collections.singletonList(0)));
-        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(Collections.singletonList(101)));
-        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(List.of(12, 40, 101)));
+        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(Collections.singletonList(100)));
+        assertThrows(IndexOutOfBoundsException.class, () -> startTextRange.split(List.of(12, 40, 100)));
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -4,13 +4,18 @@ import java.io.File;
 import java.nio.file.Path;
 import java.util.Map;
 
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.mockito.MockitoAnnotations;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
@@ -21,7 +26,10 @@ import com.knecon.fforesight.service.layoutparser.processor.services.visualizati
 import com.knecon.fforesight.service.layoutparser.server.PDFNetInitializer;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;
+import com.knecon.fforesight.tenantcommons.TenantsClient;
+import com.pdftron.pdf.PDFNet;
 
+import jakarta.annotation.PostConstruct;
 import lombok.SneakyThrows;
 
 public class ViewerDocumentTest extends BuildDocumentTest {
@@ -51,7 +59,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
 
         long start = System.currentTimeMillis();
         Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
-        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
+        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }
 
@@ -79,7 +87,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
         Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE_OLD, classificationDocument);
 
-        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
+        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -32,7 +32,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.textbloc
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
@@ -105,9 +105,9 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
         List<PageContents> textPositionPerPage = PageContentExtractor.getSortedPageContents(fileName);
         var textPositions = textPositionPerPage.stream()
-                .flatMap(t -> t.getSortedWords()
+                .flatMap(t -> t.getSortedTextPositionSequences()
                         .stream()
-                        .map(Word::toString))
+                        .map(TextPositionSequence::toString))
                 .collect(Collectors.joining(" "));
         assertThat(textPositions.contains(textToSearch)).isFalse();
 
@@ -117,7 +117,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                            .get(0).getTextBlocks().size()).isEqualTo(3);
         assertThat(classificationDocument.getHeaders()
                            .get(0).getTextBlocks()
-                           .get(0).getWords().size()).isEqualTo(8);
+                           .get(0).getSequences().size()).isEqualTo(8);
         assertThat(classificationDocument.getHeaders()
                            .get(0).getTextBlocks()
                            .get(0).toString()).contains(textToSearch);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/GapAcrossLinesDetectionServiceTest.java

@@ -36,7 +36,7 @@ class GapAcrossLinesDetectionServiceTest {
         System.out.println("start column detection");
         start = System.currentTimeMillis();
         for (PageInformation pageInformation : pageInformations) {
-            GapInformation gapInformation = GapDetectionService.findGapsInLines(pageInformation.getPageContents().getSortedWords(), pageInformation.getMainBodyTextFrame());
+            GapInformation gapInformation = GapDetectionService.findGapsInLines(pageInformation.getPageContents().getSortedTextPositionSequences(),  pageInformation.getMainBodyTextFrame());
             columnsPerPage.add(GapsAcrossLinesService.detectXGapsAcrossLines(gapInformation, pageInformation.getMainBodyTextFrame()));
         }
         System.out.printf("Finished column detection in %d ms%n", System.currentTimeMillis() - start);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/InvisibleTableDetectionServiceTest.java

@@ -10,7 +10,7 @@ import java.util.stream.Collectors;
 import org.junit.jupiter.api.Test;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.PageInformation;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.InvisibleTableDetectionService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
@@ -34,18 +34,18 @@ class InvisibleTableDetectionServiceTest {
                 .collect(Collectors.toList());
 
         int pageNumber = 1;
-        Rectangle2D tableBBox = pageContents.get(0).getPageContents().getSortedWords().subList(45, 152)
+        Rectangle2D tableBBox = pageContents.get(0).getPageContents().getSortedTextPositionSequences().subList(45, 152)
                 .stream()
-                .map(Word::getBBox)
+                .map(TextPositionSequence::getBBox)
                 .map(this::mirrorY)
                 .collect(RectangleTransformations.collectBBox());
 
-        List<Word> words = pageContents.get(0).getPageContents().getSortedWords()
+        List<TextPositionSequence> textPositionSequences = pageContents.get(0).getPageContents().getSortedTextPositionSequences()
                 .stream()
                 .filter(textPositionSequence -> tableBBox.contains(mirrorY(textPositionSequence.getBBox())))
                 .toList();
 
-        var table = InvisibleTableDetectionService.detectTable(words, tableBBox);
+        var table = InvisibleTableDetectionService.detectTable(textPositionSequences, tableBBox);
 
         PdfDraw.drawRectanglesPerPage(fileName,
                                       List.of(table.stream()

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/PageContentExtractorTest.java

@@ -6,7 +6,7 @@ import java.util.List;
 import org.junit.jupiter.api.Test;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
 
@@ -25,9 +25,9 @@ class PageContentExtractorTest {
 
         PdfDraw.drawRectanglesPerPageNumberedByLine(fileName,
                 textPositionPerPage.stream()
-                        .map(t -> t.getSortedWords()
+                        .map(t -> t.getSortedTextPositionSequences()
                                 .stream()
-                                .map(Word::getBBoxPdf)
+                                .map(TextPositionSequence::getBBoxPdf)
                                 .map(List::of)
                                 .toList())
                         .toList(), tmpFileName);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingsClassifierTest.java

@@ -11,7 +11,7 @@ import org.junit.jupiter.api.Test;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TextRulingsClassifier;
@@ -32,16 +32,16 @@ public class RulingsClassifierTest {
         for (PageContents pageContent : pageContents) {
             CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(Collections.emptyList(), pageContent.getRulings());
             RectangularIntersectionFinder.find(cleanRulings.getHorizontals(), cleanRulings.getVerticals());
-            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(pageContent.getSortedWords(), cleanRulings);
+            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(pageContent.getSortedTextPositionSequences(), cleanRulings);
 
-            assertTrue(pageContent.getSortedWords()
+            assertTrue(pageContent.getSortedTextPositionSequences()
                                .stream()
                                .filter(word -> word.toString().equals("Underlined"))
-                               .allMatch(Word::isUnderline));
-            assertTrue(pageContent.getSortedWords()
+                               .allMatch(TextPositionSequence::isUnderline));
+            assertTrue(pageContent.getSortedTextPositionSequences()
                                .stream()
                                .filter(word -> word.toString().equals("Striketrough"))
-                               .allMatch(Word::isStrikethrough));
+                               .allMatch(TextPositionSequence::isStrikethrough));
 
             assertEquals(4,
                          cleanRulings.buildAll()
@@ -70,7 +70,7 @@ public class RulingsClassifierTest {
         for (PageContents pageContent : pageContents) {
             CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(Collections.emptyList(), pageContent.getRulings());
             RectangularIntersectionFinder.find(cleanRulings.getHorizontals(), cleanRulings.getVerticals());
-            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(pageContent.getSortedWords(), cleanRulings);
+            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(pageContent.getSortedTextPositionSequences(), cleanRulings);
 
             assertEquals(30, cleanRulings.getHorizontals().size());
             assertEquals(30, cleanRulings.getTableLines().getHorizontals().size());

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -10,7 +10,9 @@ import java.util.Optional;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -53,6 +55,11 @@ public abstract class AbstractTest {
     @MockBean
     private RabbitTemplate rabbitTemplate;
 
+    @MockBean
+    private RabbitAdmin rabbitAdmin;
+
+    @MockBean
+    private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
     protected final static String ORIGIN_FILE_ID = "origin";
 
     protected final static String VISUAL_LAYOUT_FILE = "visual";
@@ -141,25 +148,16 @@ public abstract class AbstractTest {
     @SneakyThrows
     protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile, String visualLayoutParsingResponseFile) {
 
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
         ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
         ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
         ClassPathResource visualLayoutParsingResponseResource = new ClassPathResource(visualLayoutParsingResponseFile);
-        if (file.startsWith("/")) {
-            try (InputStream fileInputStream = new FileInputStream(file)) {
-                return prepareStorage(Path.of(file).getFileName().toString(),
-                                      fileInputStream,
-                                      cvServiceResponseFileResource.getInputStream(),
-                                      imageInfoFileResource.getInputStream(),
-                                      visualLayoutParsingResponseResource.getInputStream());
-            }
-        } else {
-            return prepareStorage(Path.of(file).getFileName().toString(),
-                                  new ClassPathResource(file).getInputStream(),
-                                  cvServiceResponseFileResource.getInputStream(),
-                                  imageInfoFileResource.getInputStream(),
-                                  visualLayoutParsingResponseResource.getInputStream());
-        }
 
+        return prepareStorage(Path.of(file).getFileName().toString(),
+                              pdfFileResource.getInputStream(),
+                              cvServiceResponseFileResource.getInputStream(),
+                              imageInfoFileResource.getInputStream(),
+                              visualLayoutParsingResponseResource.getInputStream());
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -1,6 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;
 
+import java.awt.geom.Rectangle2D;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.nio.file.Path;
 import java.util.Map;
 
@@ -10,11 +12,27 @@ import org.springframework.core.io.ClassPathResource;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.viewerdoc.model.Standard14EmbeddableFont;
+import com.pdftron.common.Matrix2D;
+import com.pdftron.pdf.ColorPt;
+import com.pdftron.pdf.ColorSpace;
+import com.pdftron.pdf.Element;
+import com.pdftron.pdf.ElementBuilder;
+import com.pdftron.pdf.ElementWriter;
+import com.pdftron.pdf.Font;
+import com.pdftron.pdf.GState;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.sdf.SDFDoc;
 
 import lombok.SneakyThrows;
 
@@ -27,12 +45,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
     @SneakyThrows
     protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {
 
-        File fileResource;
-        if (filename.startsWith("/")) {
-            fileResource = new File(filename);
-        } else {
-            fileResource = new ClassPathResource(filename).getFile();
-        }
+        File fileResource = new ClassPathResource(filename).getFile();
         prepareStorage(filename);
         return layoutParsingPipeline.parseLayout(layoutParsingType,
                                                  fileResource,
@@ -76,5 +89,6 @@ public abstract class BuildDocumentTest extends AbstractTest {
 
     }
 
+
 }