Merge branch 'RED-9975' into 'main'

RED-9975: improve SuperSection handling

See merge request fforesight/layout-parser!223

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -42,6 +42,19 @@ tasks.jacocoTestReport {
 }
 
 allprojects {
+
+    tasks.withType<Javadoc> {
+        options {
+            this as StandardJavadocDocletOptions
+            addBooleanOption("Xdoclint:none", true)
+            addStringOption("Xmaxwarns", "1")
+        }
+    }
+
+    pmd {
+        setConsoleOutput(true)
+    }
+
     publishing {
         publications {
             create<MavenPublication>(name) {
@@ -64,6 +77,7 @@ java {
     withJavadocJar()
 }
 
+
 repositories {
     mavenLocal()
     mavenCentral()

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -19,6 +19,16 @@ public class SimplifiedText {
     @Schema(description = "Number of pages in the entire document.")
     private int numberOfPages;
     @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
+    @Builder.Default
     private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+    @Schema(description = "A list of the main section numbers ")
+    @Builder.Default
+    private List<String> mainSectionNumbers = new ArrayList<>();
+    @Schema(description = "A list of the header section numbers ")
+    @Builder.Default
+    private List<String> headerSectionNumbers = new ArrayList<>();
+    @Schema(description = "A list of the footer section numbers ")
+    @Builder.Default
+    private List<String> footerSectionNumbers = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,6 +2,9 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
-    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE_PREFIX = "layout_parsing_request";
+    public static final String LAYOUT_PARSING_REQUEST_EXCHANGE = "layout_parsing_request_exchange";
+    public static final String LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX = "layout_parsing_response";
+    public static final String LAYOUT_PARSING_RESPONSE_EXCHANGE = "layout_parsing_response_exchange";
+    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_error";
 }

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -16,7 +16,7 @@ dependencies {
         exclude("org.springframework.boot", "spring-boot-starter-security")
         exclude("org.springframework.boot", "spring-boot-starter-validation")
     }
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.30.0")
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
 
     implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
@@ -25,7 +25,13 @@ dependencies {
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
     implementation("org.jgrapht:jgrapht-core:1.5.2")
+    implementation("org.apache.pdfbox:jbig2-imageio:3.0.4")
+    implementation("com.github.jai-imageio:jai-imageio-core:1.4.0")
+    implementation("com.github.jai-imageio:jai-imageio-jpeg2000:1.4.0")
     implementation("org.tinspin:tinspin-indexes:2.1.3")
     implementation("org.commonmark:commonmark:0.22.0")
     implementation("org.commonmark:commonmark-ext-gfm-tables:0.22.0")
+    implementation("com.pdftron:PDFNet:10.11.0")
+    implementation("org.apache.commons:commons-text:1.12.0")
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -2,13 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor;
 
 import static java.lang.String.format;
 
-import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
@@ -24,7 +24,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.No
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.markdown.MarkdownMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.MarkdownMapper;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -69,6 +69,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDF
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
 import io.micrometer.observation.Observation;
 import io.micrometer.observation.ObservationRegistry;
@@ -137,17 +138,16 @@ public class LayoutParsingPipeline {
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
         Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null //
-                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
-                                                           classificationDocument);
+                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument);
 
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
 
-        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, layoutParsingRequest.visualLayoutParsingFileId().isPresent());
+        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false);
 
         log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
 
         layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-        if(layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
+        if (layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
             layoutParsingStorageService.storeMarkdownFile(layoutParsingRequest.documentMarkdownFileStorageId().get(), new MarkdownMapper().toMarkdownContent(documentGraph));
         }
         layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
@@ -245,7 +245,7 @@ public class LayoutParsingPipeline {
         OutlineObject lastProcessedOutlineObject = null;
 
         // parsing the structure elements could be useful as well
-        if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD && layoutParsingType != LayoutParsingType.DOCUMINE_OLD) {
+        if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD) {
             classificationDocument.setOutlineObjectTree(outlineExtractorService.getOutlineObjectTree(originDocument));
         }
 
@@ -271,11 +271,13 @@ public class LayoutParsingPipeline {
             stripper.setStartPage(pageNumber);
             stripper.setEndPage(pageNumber);
             stripper.setPdpage(pdPage);
-            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
-                stripper.setSortByPosition(true);
-            }
             stripper.getText(originDocument);
             List<TextPositionSequence> words = stripper.getTextPositionSequences();
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
+                var lines = TextPositionOperations.groupByLine(new HashSet<>(words));
+                classificationDocument.getLayoutDebugLayer().addLineVisualizationsFromNestedTextPosition(lines, pageNumber);
+                words = TextPositionOperations.sortLines(lines);
+            }
             classificationDocument.getLayoutDebugLayer().addTextVisualizations(words, pageNumber);
 
             PDRectangle pdr = pdPage.getMediaBox();
@@ -321,18 +323,19 @@ public class LayoutParsingPipeline {
             classificationPage.setPageWidth(cropbox.getWidth());
             classificationPage.setPageHeight(cropbox.getHeight());
 
-            if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD && layoutParsingType != LayoutParsingType.DOCUMINE_OLD) {
-                List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber - 1, new ArrayList<>());
+            if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD) {
+                List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber, new ArrayList<>());
 
                 OutlineObject notFoundOutlineObject = null;
                 if (lastProcessedOutlineObject != null && !lastProcessedOutlineObject.isFound()) {
-                    lastProcessedOutlineObject.setPoint(new Point2D.Float(0, cropbox.getHeight()));
+                    lastProcessedOutlineObject.resetPoint();
                     notFoundOutlineObject = lastProcessedOutlineObject;
                 }
                 if (!outlineObjects.isEmpty()) {
                     classificationPage.setOutlineObjects(outlineObjects);
                     lastProcessedOutlineObject = blockificationPostprocessingService.sanitizeOutlineBlocks(classificationPage, notFoundOutlineObject);
                 }
+                classificationDocument.getLayoutDebugLayer().addOutlineObjects(outlineObjects, pageInformation);
             }
 
             classificationDocument.getLayoutDebugLayer().addMarkedContentVisualizations(stripper.getMarkedContents(), pageNumber);
@@ -376,6 +379,12 @@ public class LayoutParsingPipeline {
             case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
         }
 
+        if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
+            for (ClassificationPage page : classificationDocument.getPages()) {
+                docuMineBlockificationService.mergeblocks(page, page.getCleanRulings().withoutTextRulings(), 0, 10);
+            }
+        }
+
         List<TextPageBlock> headlines = classificationDocument.getPages()
                 .stream()
                 .flatMap(classificationPage -> classificationPage.getTextBlocks()

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/DocstrumSegmentationService.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum;
 
 import java.util.ArrayList;
+import java.util.EnumMap;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -26,6 +27,7 @@ import lombok.RequiredArgsConstructor;
 @RequiredArgsConstructor
 public class DocstrumSegmentationService {
 
+    public static final double SAME_DIRECTION_THRESHOLD = 0.9;
     private final NearestNeighbourService nearestNeighbourService;
     private final SpacingService spacingService;
     private final LineBuilderService lineBuilderService;
@@ -35,13 +37,44 @@ public class DocstrumSegmentationService {
 
     public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
 
-        List<Zone> zones = new ArrayList<>();
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.ZERO));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.QUARTER_CIRCLE));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.HALF_CIRCLE));
-        zones.addAll(computeZones(textPositions, usedRulings, visualizations, TextDirection.THREE_QUARTER_CIRCLE));
+        EnumMap<TextDirection, Integer> directionCounts = new EnumMap<>(TextDirection.class);
 
-        return readingOrderService.resolve(zones, xyOrder);
+        List<Zone> newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.ZERO);
+        directionCounts.put(TextDirection.ZERO, newZones.size());
+        List<Zone> zones = new ArrayList<>(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.QUARTER_CIRCLE);
+        directionCounts.put(TextDirection.QUARTER_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.HALF_CIRCLE);
+        directionCounts.put(TextDirection.HALF_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.THREE_QUARTER_CIRCLE);
+        directionCounts.put(TextDirection.THREE_QUARTER_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        return readingOrderService.resolve(zones, xyOrder, mostSameDirection(directionCounts));
+    }
+
+
+    private boolean mostSameDirection(EnumMap<TextDirection, Integer> directionCounts) {
+
+        int total = directionCounts.values()
+                .stream()
+                .mapToInt(i -> i).sum();
+
+        if ((double) directionCounts.get(TextDirection.ZERO) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.QUARTER_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.HALF_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.THREE_QUARTER_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        }
+        return false;
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/AngleFilter.java

@@ -15,10 +15,16 @@ public class AngleFilter {
 
     public boolean matches(Neighbor neighbor) {
 
+        return matches(neighbor.getAngle());
+    }
+
+
+    public boolean matches(double angle) {
+
         if (lowerAngle <= upperAngle) {
-            return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
+            return lowerAngle <= angle && angle < upperAngle;
         } else {
-            return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
+            return lowerAngle <= angle || angle < upperAngle;
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/BoundingBox.java

@@ -7,8 +7,12 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
 
 @Data
+@SuperBuilder
+@NoArgsConstructor
 public abstract class BoundingBox {
 
     // Java coordinate system: (0, 0) is always upper left, x is increasing left to right and y is increasing from top to bottom.
@@ -19,7 +23,7 @@ public abstract class BoundingBox {
     // This rotates completely in 90 degree steps with page rotation.
     // Needs to be used when writing to a PDF.
     // Also, these are definitely correct and should be used whenever possible.
-    protected Rectangle2D bBoxInitialUserSpace;
+    protected Rectangle2D bBoxPdf;
 
     protected static final float VERTICAL_COMPARISON_THRESHOLD = 0.4f;
 
@@ -50,25 +54,25 @@ public abstract class BoundingBox {
 
     public double getPdfMinX() {
 
-        return bBoxInitialUserSpace.getMinX();
+        return bBoxPdf.getMinX();
     }
 
 
     public double getPdfMaxX() {
 
-        return bBoxInitialUserSpace.getMaxX();
+        return bBoxPdf.getMaxX();
     }
 
 
     public double getPdfMinY() {
 
-        return bBoxInitialUserSpace.getMinY();
+        return bBoxPdf.getMinY();
     }
 
 
     public double getPdfMaxY() {
 
-        return bBoxInitialUserSpace.getMaxY();
+        return bBoxPdf.getMaxY();
     }
 
 
@@ -129,13 +133,31 @@ public abstract class BoundingBox {
     }
 
 
-    public boolean intersectsY(BoundingBox other) {
+    public boolean intersectsX(BoundingBox other, float threshold) {
+
+        return this.getX() - threshold <= other.getMaxX() && this.getMaxX() + threshold >= other.getX();
+    }
+
+
+    public boolean intersectsPdf(BoundingBox other) {
+
+        return this.intersectsXPdf(other) && this.intersectsYPdf(other);
+    }
+
+
+    public boolean intersectsPdf(BoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsXPdf(other, xThreshold) && this.intersectsYPdf(other, yThreshold);
+    }
+
+
+    public boolean intersectsYPdf(BoundingBox other) {
 
         return this.getPdfMinY() <= other.getPdfMaxY() && this.getPdfMaxY() >= other.getPdfMinY();
     }
 
 
-    public boolean intersectsYJava(BoundingBox other) {
+    public boolean intersectsY(BoundingBox other) {
 
         return this.getY() <= other.getMaxY() && this.getMaxY() >= other.getY();
     }
@@ -143,25 +165,31 @@ public abstract class BoundingBox {
 
     public boolean intersectsY(BoundingBox other, float threshold) {
 
+        return this.getY() - threshold <= other.getMaxY() && this.getMaxY() + threshold >= other.getY();
+    }
+
+
+    public boolean intersectsYPdf(BoundingBox other, float threshold) {
+
         return this.getPdfMinY() - threshold <= other.getPdfMaxY() && this.getPdfMaxY() + threshold >= other.getPdfMinY();
     }
 
 
-    public boolean intersectsX(BoundingBox other) {
+    public boolean intersectsXPdf(BoundingBox other) {
 
         return this.getPdfMinX() <= other.getPdfMaxX() && this.getPdfMaxX() >= other.getPdfMinX();
     }
 
 
-    public boolean intersectsXJava(BoundingBox other) {
+    public boolean intersectsX(BoundingBox other) {
 
         return this.getX() <= other.getMaxX() && this.getMaxX() >= other.getMinX();
     }
 
 
-    public boolean intersectsX(BoundingBox other, float threshold) {
+    public boolean intersectsXPdf(BoundingBox other, float threshold) {
 
-        return this.getPdfMinX() - threshold <= other.getMaxX() && this.getMaxX() + threshold >= other.getPdfMinX();
+        return this.getPdfMinX() - threshold <= other.getPdfMaxX() && this.getMaxX() + threshold >= other.getPdfMinX();
     }
 
 
@@ -170,8 +198,8 @@ public abstract class BoundingBox {
         this.bBox = components.stream()
                 .map(BoundingBox::getBBox)
                 .collect(RectangleTransformations.collectBBox());
-        this.bBoxInitialUserSpace = components.stream()
-                .map(BoundingBox::getBBoxInitialUserSpace)
+        this.bBoxPdf = components.stream()
+                .map(BoundingBox::getBBoxPdf)
                 .collect(RectangleTransformations.collectBBox());
     }
 
@@ -229,25 +257,25 @@ public abstract class BoundingBox {
 
     public boolean rightOf(BoundingBox other) {
 
-        return this.intersectsYJava(other) && other.getMaxX() <= this.getMinX();
+        return this.intersectsY(other) && other.getMaxX() <= this.getMinX();
     }
 
 
     public boolean leftOf(BoundingBox other) {
 
-        return this.intersectsYJava(other) && other.getMinX() >= this.getMaxX();
+        return this.intersectsY(other) && other.getMinX() >= this.getMaxX();
     }
 
 
     public boolean isAbove(BoundingBox other) {
 
-        return this.intersectsXJava(other) && other.getMinY() >= this.getMaxY();
+        return this.intersectsX(other) && other.getMinY() >= this.getMaxY();
     }
 
 
     public boolean isBelow(BoundingBox other) {
 
-        return this.intersectsXJava(other) && this.getMinY() >= other.getMaxY();
+        return this.intersectsX(other) && this.getMinY() >= other.getMaxY();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Character.java

@@ -35,7 +35,7 @@ public class Character {
 
     public double getHeight() {
 
-        return textPosition.getHeightDir();
+        return textPosition.getHeightDirAdj();
     }
 
 
@@ -65,9 +65,9 @@ public class Character {
         double s = Math.sin(-0);
         double c = Math.cos(-0);
         xs[0] = c * x - s * y;
-        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
+        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDirAdj());
         xs[2] = c * other.x - s * other.y;
-        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
+        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDirAdj());
         boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
         Arrays.sort(xs);
         return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -1,18 +1,28 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.BOLD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.BOLD_ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence.STANDARD;
+
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 
 @Data
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
-public class Line extends BoundingBox {
+@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
+public class Line extends TextBoundingBox {
 
     private static final double WORD_DISTANCE_MULTIPLIER = 0.18;
 
@@ -28,6 +38,8 @@ public class Line extends BoundingBox {
 
     private final double height;
 
+    private FontStyle fontStyle;
+
     private final List<Character> characters;
     private final List<TextPositionSequence> words = new ArrayList<>();
 
@@ -67,6 +79,29 @@ public class Line extends BoundingBox {
         height = computeHeight();
         computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
         buildBBox();
+        computeFontStyle();
+    }
+
+
+    private void computeFontStyle() {
+
+        EnumMap<FontStyle, AtomicInteger> fontStyleCounter = new EnumMap<>(FontStyle.class);
+        for (FontStyle fontStyle : FontStyle.values()) {
+            fontStyleCounter.put(fontStyle, new AtomicInteger(0));
+        }
+        for (TextPositionSequence word : words) {
+            switch (word.getFontStyle()) {
+                case STANDARD -> fontStyleCounter.get(FontStyle.REGULAR).getAndIncrement();
+                case BOLD -> fontStyleCounter.get(FontStyle.BOLD).getAndIncrement();
+                case ITALIC -> fontStyleCounter.get(FontStyle.ITALIC).getAndIncrement();
+                case BOLD_ITALIC -> fontStyleCounter.get(FontStyle.BOLD_ITALIC).getAndIncrement();
+            }
+        }
+        fontStyle = fontStyleCounter.entrySet()
+                .stream()
+                .max(Comparator.comparing(entry -> entry.getValue().get()))
+                .map(Map.Entry::getKey)
+                .orElse(FontStyle.REGULAR);
     }
 
 
@@ -144,8 +179,8 @@ public class Line extends BoundingBox {
     private void buildBBox() {
 
         this.setToBBoxOfComponents(characters.stream()
-                                       .map(Character::getTextPosition)
-                                       .toList());
+                                           .map(Character::getTextPosition)
+                                           .toList());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/TextBoundingBox.java

@@ -0,0 +1,102 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.SuperBuilder;
+
+@Getter
+@Setter
+@SuperBuilder
+@NoArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+public abstract class TextBoundingBox extends BoundingBox {
+
+    protected Rectangle2D bBoxDirAdj;
+
+    protected TextDirection dir;
+
+
+    @Override
+    public void setToBBoxOfComponents(List<? extends BoundingBox> components) {
+
+        super.setToBBoxOfComponents(components);
+        this.bBoxDirAdj = components.stream()
+                .filter(c -> c instanceof TextBoundingBox)
+                .map(c -> (TextBoundingBox) c)
+                .map(TextBoundingBox::getBBoxDirAdj)
+                .collect(RectangleTransformations.collectBBox());
+
+        Set<TextDirection> textDirections = components.stream()
+                .filter(c -> c instanceof TextBoundingBox)
+                .map(c -> (TextBoundingBox) c)
+                .map(TextBoundingBox::getDir)
+                .collect(Collectors.toSet());
+
+        if (textDirections.isEmpty()) {
+            dir = TextDirection.ZERO;
+        } else if (textDirections.size() > 1) {
+            throw new IllegalArgumentException("More than one text direction found");
+        } else {
+            dir = textDirections.iterator().next();
+        }
+    }
+
+
+    public double getXDirAdj() {
+
+        return this.bBoxDirAdj.getX();
+    }
+
+
+    public double getYDirAdj() {
+
+        return this.bBoxDirAdj.getY();
+    }
+
+
+    public double getWidthDirAdj() {
+
+        return this.bBoxDirAdj.getWidth();
+    }
+
+
+    public double getHeightDirAdj() {
+
+        return this.bBoxDirAdj.getHeight();
+    }
+
+
+    public double getMaxXDirAdj() {
+
+        return this.bBoxDirAdj.getMaxX();
+    }
+
+
+    public double getMaxYDirAdj() {
+
+        return this.bBoxDirAdj.getMaxY();
+    }
+
+
+    public double getCenterYDirAdj() {
+
+        return this.bBoxDirAdj.getCenterY();
+    }
+
+
+    public double getCenterXDirAdj() {
+
+        return this.bBoxDirAdj.getCenterX();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Zone.java

@@ -6,9 +6,11 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 
 @Data
-public class Zone extends BoundingBox {
+@EqualsAndHashCode(callSuper = false)
+public class Zone extends TextBoundingBox {
 
     private List<Line> lines;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ReadingOrderService.java

@@ -1,15 +1,17 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 
@@ -19,21 +21,30 @@ public class ReadingOrderService {
     private static final double THRESHOLD = 5;
     public static final double MULTI_COLUMN_DETECTION_THRESHOLD = 1.5;
 
+    private static final Comparator<TextBoundingBox> COMPARATOR = //
+            Comparator.comparing(TextBoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(TextBoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD));
 
-    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder) {
+    private static final Comparator<TextBoundingBox> COMPARATOR_DIR_ADJ = //
+            Comparator.comparing(TextBoundingBox::getYDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD));
+
+
+    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder, boolean useDirAdjCoords) {
 
         if (zones.isEmpty() || zones.size() == 1) {
             return zones;
         }
 
         if (xyReadingOrder) {
-            return resolveSingleColumnReadingOrder(zones);
+            return resolveSingleColumnReadingOrder(zones, useDirAdjCoords);
         }
 
         Map<Long, Integer> histogram = new HashMap<>();
         for (Zone zone : zones) {
-            long minY = Math.round(zone.getBBox().getMinY());
-            long maxY = Math.round(zone.getBBox().getMaxY());
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            long minY = Math.round(bbox.getMinY());
+            long maxY = Math.round(bbox.getMaxY());
             for (long i = minY; i <= maxY; i++) {
                 histogram.put(i, histogram.getOrDefault(i, 0) + 1);
             }
@@ -43,24 +54,32 @@ public class ReadingOrderService {
                     .stream()
                     .mapToInt(Integer::intValue).average()
                     .orElse(1) < MULTI_COLUMN_DETECTION_THRESHOLD) {
-            return resolveSingleColumnReadingOrder(zones);
+            return resolveSingleColumnReadingOrder(zones, useDirAdjCoords);
         } else {
 
-            return resolveMultiColumnReadingOder(zones);
+            return resolveMultiColumnReadingOder(zones, useDirAdjCoords);
         }
 
     }
 
 
-    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones) {
+    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones, boolean useDirAdjCoords) {
 
-        zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                           .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+        if (useDirAdjCoords) {
+            return zones.stream()
+                    .collect(Collectors.groupingBy(TextBoundingBox::getDir)).values()
+                    .stream()
+                    .flatMap(words -> words.stream()
+                            .sorted(COMPARATOR_DIR_ADJ))
+                    .toList();
+        }
+
+        zones.sort(COMPARATOR);
         return zones;
     }
 
 
-    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
+    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones, boolean useDirAdjCoords) {
 
         // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
         // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
@@ -69,11 +88,12 @@ public class ReadingOrderService {
         double maxX = Double.NEGATIVE_INFINITY;
 
         for (Zone zone : zones) {
-            if (zone.getX() < minX) {
-                minX = zone.getX();
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            if (bbox.getX() < minX) {
+                minX = zone.getXDirAdj();
             }
-            if (zone.getX() + zone.getWidth() > maxX) {
-                maxX = zone.getX() + zone.getWidth();
+            if (bbox.getMaxX() > maxX) {
+                maxX = zone.getMaxXDirAdj();
             }
         }
 
@@ -82,24 +102,27 @@ public class ReadingOrderService {
         List<Zone> leftOf = new ArrayList<>();
         List<Zone> rightOf = new ArrayList<>();
         List<Zone> middle = new ArrayList<>();
+
         for (Zone zone : zones) {
-            if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            if (bbox.getX() < midLineXCoordinate && bbox.getX() + bbox.getWidth() < midLineXCoordinate) {
                 leftOf.add(zone);
-            } else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
+            } else if (bbox.getX() > midLineXCoordinate && bbox.getX() + bbox.getWidth() > midLineXCoordinate) {
                 rightOf.add(zone);
             } else {
                 middle.add(zone);
             }
         }
 
-        leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                            .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                             .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                            .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+        if (useDirAdjCoords) {
+            leftOf.sort(COMPARATOR_DIR_ADJ);
+            rightOf.sort(COMPARATOR_DIR_ADJ);
+            middle.sort(COMPARATOR_DIR_ADJ);
+        } else {
+            leftOf.sort(COMPARATOR);
+            rightOf.sort(COMPARATOR);
+            middle.sort(COMPARATOR);
+        }
 /*
         List<Zone> leftNotIntersecting = new ArrayList<>();
         for (Zone leftZone : leftOf) {
@@ -151,8 +174,9 @@ public class ReadingOrderService {
 
         while (itty.hasNext()) {
             Zone current = itty.next();
+            Rectangle2D bbox = useDirAdjCoords ? current.getBBoxDirAdj() : current.getBBox();
             for (int i = 0; i < sortedZones.size(); i++) {
-                if (current.getY() < sortedZones.get(i).getY()) {
+                if (bbox.getY() < sortedZones.get(i).getY()) {
                     sortedZones.add(i, current);
                     itty.remove();
                     break;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ZoneBuilderService.java

@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
+import static com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier.numericalIdentifierPattern;
+
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -21,7 +23,7 @@ public class ZoneBuilderService {
     private static final double MIN_HORIZONTAL_DISTANCE_MULTIPLIER = -0.5;
     private static final double MAX_VERTICAL_DISTANCE_MULTIPLIER = 1.2;
 
-    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -3.0;
+    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -7;
 
     private static final double MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER = 0.5;
 
@@ -38,7 +40,7 @@ public class ZoneBuilderService {
 
         double minHorizontalDistance = characterSpacing * MIN_HORIZONTAL_DISTANCE_MULTIPLIER;
         double maxVerticalDistance = lineSpacing * MAX_VERTICAL_DISTANCE_MULTIPLIER;
-        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
+        double minHorizontalMergeDistance = lineSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
         double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;
 
         UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
@@ -54,11 +56,26 @@ public class ZoneBuilderService {
                     return;
                 }
 
-                double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
-                scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
+//                if (!innerLine.getFontStyle().equals(outerLine.getFontStyle()) //
+//                    && !outerLine.intersectsY(innerLine, -2f)) {
+//                    return;
+//                }
 
-                double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
-                double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
+                double horizontalScale = Math.min(outerLine.getHeightDirAdj(), innerLine.getHeightDirAdj()) / meanHeight;
+                horizontalScale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(horizontalScale, MAX_LINE_SIZE_SCALE));
+                double verticalScale = horizontalScale;
+
+//                if (innerLine.toString().endsWith(":")
+//                    || outerLine.toString().endsWith(":")
+//                    || numericalIdentifierPattern.matcher(innerLine.toString()).matches()
+//                    || numericalIdentifierPattern.matcher(outerLine.toString()).matches()) {
+//
+//                    horizontalScale *= 5;
+//                    verticalScale /= 10;
+//                }
+
+                double horizontalDistance = outerLine.horizontalDistance(innerLine) / horizontalScale;
+                double verticalDistance = outerLine.verticalDistance(innerLine) / verticalScale;
 
                 if ((!(minHorizontalDistance <= horizontalDistance) || !(verticalDistance <= maxVerticalDistance)) //
                     && (!(minHorizontalMergeDistance <= horizontalDistance) || !(verticalDistance <= maxVerticalMergeDistance))) {
@@ -87,7 +104,7 @@ public class ZoneBuilderService {
         double weights = 0.0;
         for (Line line : lines) {
             double weight = line.getLength();
-            meanHeight += line.getHeight() * weight;
+            meanHeight += line.getHeightDirAdj() * weight;
             weights += weight;
         }
         meanHeight /= weights;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/utils/DoubleUtils.java

@@ -7,9 +7,12 @@ public class DoubleUtils {
         if (Double.isNaN(d1) || Double.isNaN(d2)) {
             return Double.compare(d1, d2);
         }
-        long i1 = Math.round(d1 / (precision == 0 ? 1 : precision));
-        long i2 = Math.round(d2 / (precision == 0 ? 1 : precision));
-        return Long.compare(i1, i2);
+
+        if (Math.abs(d1 - d2) < precision) {
+            return 0;
+        }
+
+        return Double.compare(d1, d2);
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java

@@ -18,6 +18,7 @@ import lombok.RequiredArgsConstructor;
 
 @Data
 @RequiredArgsConstructor
+
 public class ClassificationPage {
 
     @NonNull
@@ -25,7 +26,7 @@ public class ClassificationPage {
 
     private List<OutlineObject> outlineObjects = new ArrayList<>();
 
-    private List<AbstractPageBlock> headlines  = new ArrayList<>();
+    private List<AbstractPageBlock> headlines = new ArrayList<>();
 
     private List<ClassifiedImage> images = new ArrayList<>();
 
@@ -44,7 +45,7 @@ public class ClassificationPage {
     private float pageWidth;
     private float pageHeight;
 
-    CleanRulings cleanRulings;
+    private CleanRulings cleanRulings;
 
     private Map<String, List<Rectangle2D>> markedContentBboxPerType = new HashMap<>();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/FloatFrequencyCounter.java

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -12,10 +11,14 @@ import lombok.Getter;
 @Getter
 public class FloatFrequencyCounter {
 
-    Map<Float, Integer> countPerValue = new HashMap<>();
+    Map<Double, Integer> countPerValue = new HashMap<>();
+    boolean changed;
+    Double mostPopularCache;
 
 
-    public void add(float value) {
+    public void add(double value) {
+
+        changed = true;
 
         if (!countPerValue.containsKey(value)) {
             countPerValue.put(value, 1);
@@ -25,9 +28,11 @@ public class FloatFrequencyCounter {
     }
 
 
-    public void addAll(Map<Float, Integer> otherCounter) {
+    public void addAll(Map<Double, Integer> otherCounter) {
 
-        for (Map.Entry<Float, Integer> entry : otherCounter.entrySet()) {
+        changed = true;
+
+        for (Map.Entry<Double, Integer> entry : otherCounter.entrySet()) {
             if (countPerValue.containsKey(entry.getKey())) {
                 countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
             } else {
@@ -37,36 +42,36 @@ public class FloatFrequencyCounter {
     }
 
 
-    public Float getMostPopular() {
+    public Double getMostPopular() {
 
-        Map.Entry<Float, Integer> mostPopular = null;
-        for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
-                mostPopular = entry;
+        if (changed) {
+            Map.Entry<Double, Integer> mostPopular = null;
+            for (Map.Entry<Double, Integer> entry : countPerValue.entrySet()) {
+                if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
+                    mostPopular = entry;
+                }
             }
+            mostPopularCache = mostPopular != null ? mostPopular.getKey() : null;
+            changed = false;
         }
-        return mostPopular != null ? mostPopular.getKey() : null;
+
+        return mostPopularCache;
     }
 
 
-    public List<Float> getHighterThanMostPopular() {
+    public List<Double> getValuesInReverseOrder() {
 
-        Float mostPopular = getMostPopular();
-        List<Float> higher = new ArrayList<>();
-        for (Float value : countPerValue.keySet()) {
-            if (value > mostPopular) {
-                higher.add(value);
-            }
-        }
-
-        return higher.stream().sorted(Collections.reverseOrder()).collect(Collectors.toList());
+        return countPerValue.keySet()
+                .stream()
+                .sorted(Collections.reverseOrder())
+                .collect(Collectors.toList());
     }
 
 
-    public Float getHighest() {
+    public Double getHighest() {
 
-        Float highest = null;
-        for (Float value : countPerValue.keySet()) {
+        Double highest = null;
+        for (Double value : countPerValue.keySet()) {
             if (highest == null || value > highest) {
                 highest = value;
             }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/SectionIdentifier.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -15,11 +16,13 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {
 
-    static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?");
 
     public enum Format {
         EMPTY,
         NUMERICAL,
+        ALPHANUMERIC,
         DOCUMENT
     }
 
@@ -41,6 +44,10 @@ public class SectionIdentifier {
         if (numericalIdentifierMatcher.find()) {
             return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher);
         }
+        Matcher alphanumericIdentifierMatcher = alphanumericIdentifierPattern.matcher(headline);
+        if (alphanumericIdentifierMatcher.find()) {
+            return buildAlphanumericSectionIdentifier(headline, alphanumericIdentifierMatcher);
+        }
         // more formats here
         return SectionIdentifier.empty();
     }
@@ -75,7 +82,36 @@ public class SectionIdentifier {
             }
             identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
         }
-        return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false);
+        return new SectionIdentifier(Format.NUMERICAL,
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
+    }
+
+
+    private static SectionIdentifier buildAlphanumericSectionIdentifier(String headline, Matcher alphanumericIdentifierMatcher) {
+
+        String identifierString = headline.substring(alphanumericIdentifierMatcher.start(), alphanumericIdentifierMatcher.end());
+
+        String alphanumericIdentifier = alphanumericIdentifierMatcher.group(0).substring(0, 1).toUpperCase(Locale.ENGLISH);
+        int mappedCharacterValue = alphanumericIdentifier.charAt(0) - 'A' + 1;
+        List<Integer> identifiers = new LinkedList<>();
+        identifiers.add(mappedCharacterValue);
+
+        for (int i = 1; i <= 3; i++) {
+            String numericalIdentifier = alphanumericIdentifierMatcher.group(i);
+            if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) {
+                break;
+            }
+            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
+        }
+
+        return new SectionIdentifier(Format.ALPHANUMERIC,
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
     }
 
 
@@ -123,4 +159,22 @@ public class SectionIdentifier {
         return identifierString;
     }
 
+
+    public boolean isEmpty() {
+
+        return this.format.equals(Format.EMPTY);
+    }
+
+
+    public int level() {
+
+        return identifiers.size();
+    }
+
+
+    protected List<Integer> getIdentifiers() {
+
+        return identifiers;
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/AbstractNodeVisitor.java

@@ -84,7 +84,7 @@ public abstract class AbstractNodeVisitor implements NodeVisitor {
     }
 
 
-    private void visitChildren(SemanticNode semanticNode) {
+    protected void visitChildren(SemanticNode semanticNode) {
 
         semanticNode.streamChildren()
                 .forEach(node -> node.accept(this));

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/DuplicatedParagraph.java

@@ -25,11 +25,4 @@ public class DuplicatedParagraph extends Paragraph {
 
     }
 
-
-    @Override
-    public String toString() {
-
-        return super.toString();
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Page.java

@@ -1,12 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Stream;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.TextEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
@@ -29,9 +32,8 @@ public class Page {
     Integer height;
     Integer width;
     Integer rotation;
-
     @EqualsAndHashCode.Exclude
-    List<SemanticNode> mainBody;
+    List<AtomicTextBlock> textBlocksOnPage;
     @EqualsAndHashCode.Exclude
     Header header;
     @EqualsAndHashCode.Exclude
@@ -53,20 +55,44 @@ public class Page {
                 .width((int) classificationPage.getPageWidth())
                 .number(classificationPage.getPageNumber())
                 .rotation(classificationPage.getRotation())
-                .mainBody(new LinkedList<>())
+                .textBlocksOnPage(new LinkedList<>())
                 .build();
     }
 
 
+    /**
+     * Constructs and returns a {@link TextBlock} representing the concatenated text of all leaf semantic nodes in the main body.
+     *
+     * @return The main body text block.
+     */
     public TextBlock getMainBodyTextBlock() {
 
-        return mainBody.stream()
-                .filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getLeafTextBlock)
+        return textBlocksOnPage.stream()
+                .filter(atb -> !atb.isEmpty())
                 .collect(new TextBlockCollector());
     }
 
 
+    public List<SemanticNode> getMainBody() {
+
+        return textBlocksOnPage.stream()
+                .map(AtomicTextBlock::getParent)
+                .map(this::getHighestParentOnPage)
+                .distinct()
+                .toList();
+    }
+
+
+    private SemanticNode getHighestParentOnPage(SemanticNode node) {
+
+        SemanticNode currentNode = node;
+        while (currentNode.hasParent() && currentNode.getParent().onlyOnPage(this)) {
+            currentNode = currentNode.getParent();
+        }
+        return currentNode;
+    }
+
+
     @Override
     public String toString() {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java

@@ -74,7 +74,8 @@ public interface SemanticNode {
 
         return getTextBlock().getPages()
                 .stream()
-                .min(Comparator.comparingInt(Page::getNumber)).orElseThrow(() -> new IllegalStateException("SemanticNode has no Page!"));
+                .min(Comparator.comparingInt(Page::getNumber))
+                .orElseThrow(() -> new IllegalStateException("SemanticNode has no Page!"));
     }
 
 
@@ -504,4 +505,17 @@ public interface SemanticNode {
 
     void accept(NodeVisitor visitor);
 
+
+    /**
+     * Checks wether this SemanticNode appears on a single page only, and if that page is the provided one.
+     *
+     * @param page the page to check
+     * @return true, when SemanticNode is on a single page only and the page is the provided page. Otherwise, false.
+     */
+    default boolean onlyOnPage(Page page) {
+
+        Set<Page> pages = getPages();
+        return pages.size() == 1 && pages.contains(page);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/AtomicTextBlock.java

@@ -145,10 +145,7 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData,
-                                                          DocumentPositionData documentPositionData,
-                                                          SemanticNode parent,
-                                                          Page page) {
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData, DocumentPositionData documentPositionData, SemanticNode parent, Page page) {
 
         return AtomicTextBlock.builder()
                 .id(documentTextData.getId())
@@ -156,8 +153,10 @@ public class AtomicTextBlock implements TextBlock {
                 .page(page)
                 .textRange(new TextRange(documentTextData.getStart(), documentTextData.getEnd()))
                 .searchText(documentTextData.getSearchText())
-                .lineBreaks(Arrays.stream(documentTextData.getLineBreaks()).boxed().toList())
-                .stringIdxToPositionIdx(Arrays.stream(documentPositionData.getStringIdxToPositionIdx()).boxed().toList())
+                .lineBreaks(Arrays.stream(documentTextData.getLineBreaks()).boxed()
+                                    .toList())
+                .stringIdxToPositionIdx(Arrays.stream(documentPositionData.getStringIdxToPositionIdx()).boxed()
+                                                .toList())
                 .positions(toRectangle2DList(documentPositionData.getPositions()))
                 .parent(parent)
                 .build();
@@ -166,7 +165,9 @@ public class AtomicTextBlock implements TextBlock {
 
     private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
 
-        return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList();
+        return Arrays.stream(positions)
+                .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+                .toList();
     }
 
 
@@ -176,6 +177,9 @@ public class AtomicTextBlock implements TextBlock {
             throw new IndexOutOfBoundsException(format("line %d out of range for AtomicTextBlock with %d lines", lineNumber, numberOfLines()));
         }
         if (lineNumber == 0) {
+            if (lineBreaks.isEmpty()) {
+                return searchText;
+            }
             return subSequence(textRange.start(), lineBreaks.get(0) + textRange.start());
         } else if (lineNumber == numberOfLines() - 1) {
             return subSequence(lineBreaks.get(lineBreaks.size() - 1) + textRange.start(), textRange.end());
@@ -195,9 +199,9 @@ public class AtomicTextBlock implements TextBlock {
     public int getNextLinebreak(int fromIndex) {
 
         return lineBreaks.stream()//
-                .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
-                .findFirst() //
-                .orElse(searchText.length()) + textRange.start();
+                       .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
+                       .findFirst() //
+                       .orElse(searchText.length()) + textRange.start();
     }
 
 
@@ -205,9 +209,9 @@ public class AtomicTextBlock implements TextBlock {
     public int getPreviousLinebreak(int fromIndex) {
 
         return lineBreaks.stream()//
-                .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
-                .reduce((a, b) -> b)//
-                .orElse(0) + textRange.start();
+                       .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
+                       .reduce((a, b) -> b)//
+                       .orElse(0) + textRange.start();
     }
 
 
@@ -255,7 +259,10 @@ public class AtomicTextBlock implements TextBlock {
 
     protected List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {
 
-        return getLineBreaks().stream().map(linebreak -> linebreak + this.textRange.start()).filter(textRange::contains).toList();
+        return getLineBreaks().stream()
+                .map(linebreak -> linebreak + this.textRange.start())
+                .filter(textRange::contains)
+                .toList();
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java

@@ -10,8 +10,8 @@ import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 
 @Data
-@RequiredArgsConstructor
 @AllArgsConstructor
+@RequiredArgsConstructor
 public class ClassifiedImage {
 
     @NonNull
@@ -20,11 +20,18 @@ public class ClassifiedImage {
     private ImageType imageType;
     private boolean sourceByAi;
     private boolean isAppendedToSection;
-    @NonNull
     private boolean hasTransparency;
-    @NonNull
     private int page;
-    @NonNull
     private String representation;
 
+
+    public ClassifiedImage(@NonNull Rectangle2D position, @NonNull ImageType imageType, boolean hasTransparency, int page, String representation) {
+
+        this.position = position;
+        this.imageType = imageType;
+        this.hasTransparency = hasTransparency;
+        this.page = page;
+        this.representation = representation;
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineExtractorService.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
+import java.awt.geom.AffineTransform;
 import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -26,6 +27,9 @@ import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocume
 import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
@@ -82,11 +86,20 @@ public class OutlineExtractorService {
 
         String title = item.getTitle();
 
-        PDPage page = item.findDestinationPage(document);
-        if (page == null) {
+        PDPage page;
+        try {
+            // Can throw: "Error: can't convert to Destination COSArray" for some OCR'd PDFs
+            page = item.findDestinationPage(document);
+            if (page == null) {
+                return Optional.empty();
+            }
+        } catch (IOException e) {
+            log.info(String.format("Error occurred during position resolution for outline item with title %s: " + e, title));
             return Optional.empty();
         }
-        int pageNumber = document.getPages().indexOf(page);
+
+        int pageNumber = document.getPages().indexOf(page) + 1;
+        AffineTransform userSpaceToPageCoords = CoordinateTransforms.calculateInitialUserSpaceCoordsToPageCoords(PageInformation.fromPDPage(pageNumber, page));
 
         Optional<Point2D> outlinePosition = Optional.empty();
 
@@ -115,8 +128,15 @@ public class OutlineExtractorService {
             log.info(String.format("Error occurred during position resolution for outline item on page %s with title %s: " + e, pageNumber, title));
         }
 
-        return Optional.of(new OutlineObjectTreeNode(new OutlineObject(title, pageNumber, outlinePosition.orElse(new Point2D.Float(0, 0)), depth)));
+        return Optional.of(new OutlineObjectTreeNode(new OutlineObject(title,
+                                                                       pageNumber,
+                                                                       transformPointToPageCoords(outlinePosition, userSpaceToPageCoords), depth)));
+    }
 
+
+    private static Point2D transformPointToPageCoords(Optional<Point2D> outlinePosition, AffineTransform userSpaceToPageCoords) {
+
+        return outlinePosition.map(point -> userSpaceToPageCoords.transform(point, null)).orElse(null);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineObject.java

@@ -1,27 +1,34 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
 import java.awt.geom.Point2D;
+import java.util.Optional;
 
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+
+import lombok.Getter;
+import lombok.Setter;
 
-@Data
-@RequiredArgsConstructor
-@AllArgsConstructor
 public class OutlineObject {
 
+    @Getter
     private final String title;
+    @Getter
     private final int pageNumber;
-    private Point2D point;
+    @Getter
     private final int treeDepth;
 
+    private Point2D point; // java coordinates, (0, 0) is always top left
+
+    @Getter
+    @Setter
     private boolean found;
 
 
     public OutlineObject(String title, int pageNumber, Point2D point2D, int depth) {
 
-        this(title, pageNumber, depth);
+        this.title = title;
+        this.pageNumber = pageNumber;
+        this.treeDepth = depth;
         this.point = point2D;
     }
 
@@ -32,4 +39,39 @@ public class OutlineObject {
         return "OutlineObject{" + "title='" + title + '\'' + '}';
     }
 
+
+    public Optional<Point2D> getPoint() {
+
+        return Optional.ofNullable(point);
+    }
+
+
+    public boolean isAbove(BoundingBox boundingBox) {
+
+        if (point == null) {
+            return true;
+        }
+        return point.getY() <= boundingBox.getMaxY();
+    }
+
+
+    public double distance(BoundingBox boundingBox) {
+
+        if (point == null) {
+            return 0;
+        }
+        if (boundingBox.getBBox().contains(point)) {
+            return 0;
+        }
+        double deltaX = Math.min(Math.abs(boundingBox.getMinX() - point.getX()), Math.abs(boundingBox.getMaxX() - point.getX()));
+        double deltaY = Math.min(Math.abs(boundingBox.getMinY() - point.getY()), Math.abs(boundingBox.getMaxY() - point.getY()));
+        return Math.sqrt(deltaX * deltaX + deltaY * deltaY);
+    }
+
+
+    public void resetPoint() {
+
+        this.point = null;
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineObjectTree.java

@@ -39,4 +39,28 @@ public class OutlineObjectTree {
         }
     }
 
+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("OutlineObjectTree(\n");
+        for (OutlineObjectTreeNode node : rootNodes) {
+            buildString(node, sb, 1);
+        }
+        sb.append(")");
+        return sb.toString();
+    }
+
+    private void buildString(OutlineObjectTreeNode node, StringBuilder sb, int depth) {
+
+        for (int i = 0; i < depth; i++) {
+            sb.append("  ");
+        }
+        sb.append(node.getOutlineObject().getTitle()).append("\n");
+
+        for (OutlineObjectTreeNode child : node.getChildren()) {
+            buildString(child, sb, depth + 1);
+        }
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/TOCEnrichmentService.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -167,7 +168,7 @@ public class TOCEnrichmentService {
             }
         }
 
-        if (!startBlocks.isEmpty()) {
+        if (!startBlocks.isEmpty() || !startImages.isEmpty()) {
             TableOfContentItem unassigned = new TableOfContentItem(null);
             unassigned.setSectionBlocks(startBlocks);
             unassigned.setImages(startImages);
@@ -185,12 +186,8 @@ public class TOCEnrichmentService {
             List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
             List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
             // Allow merging of tables if header row is separated from first logical non-header row
-            if (previousTableNonHeaderRow.isEmpty()
-                && previousTable.getRowCount() == 1
-                && previousTable.getRows()
-                           .get(0).size() == tableNonHeaderRow.size()) {
-                previousTableNonHeaderRow = previousTable.getRows()
-                        .get(0)
+            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows().get(0).size() == tableNonHeaderRow.size()) {
+                previousTableNonHeaderRow = previousTable.getRows().get(0)
                         .stream()
                         .map(cell -> {
                             Cell fakeCell = Cell.copy(cell);
@@ -201,8 +198,7 @@ public class TOCEnrichmentService {
             }
             if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
                 for (int i = currentTable.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-                    List<Cell> row = currentTable.getRows()
-                            .get(i);
+                    List<Cell> row = currentTable.getRows().get(i);
                     if (row.size() == tableNonHeaderRow.size() && row.stream()
                             .allMatch(cell -> cell.getHeaderCells().isEmpty())) {
                         for (int j = 0; j < row.size(); j++) {
@@ -225,18 +221,15 @@ public class TOCEnrichmentService {
 
         return table.getRows()
                 .stream()
-                .flatMap(row -> row.stream()
-                        .filter(cell -> !cell.getHeaderCells().isEmpty()))
-                .findAny().isEmpty();
-
+                .flatMap(Collection::stream)
+                .allMatch(cell -> cell.getHeaderCells().isEmpty());
     }
 
 
     private List<Cell> getRowWithNonHeaderCells(TablePageBlock table) {
 
         for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
-            List<Cell> row = table.getRows()
-                    .get(i);
+            List<Cell> row = table.getRows().get(i);
             if (row.size() == 1) {
                 continue;
             }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/TableOfContentItem.java

@@ -5,7 +5,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
@@ -24,7 +24,7 @@ public class TableOfContentItem {
     private List<AbstractPageBlock> sectionBlocks = new ArrayList<>();
     private List<ClassifiedImage> images = new ArrayList<>();
 
-    private AbstractSemanticNode section;
+    private GenericSemanticNode section;
 
 
     public TableOfContentItem(TextPageBlock headline) {
@@ -45,8 +45,7 @@ public class TableOfContentItem {
         if (parent != null) {
             int index = parent.getChildren().indexOf(this);
             if (index > 0) {
-                return parent.getChildren()
-                        .get(index - 1);
+                return parent.getChildren().get(index - 1);
             }
         }
         return null;
@@ -58,8 +57,7 @@ public class TableOfContentItem {
         if (parent != null) {
             int index = parent.getChildren().indexOf(this);
             if (index >= 0 && index < parent.getChildren().size() - 1) {
-                return parent.getChildren()
-                        .get(index + 1);
+                return parent.getChildren().get(index + 1);
             }
         }
         return null;
@@ -93,17 +91,19 @@ public class TableOfContentItem {
         return false;
     }
 
+
     public List<AbstractPageBlock> getNonEmptySectionBlocks() {
 
-        return sectionBlocks.stream().filter(pageBlock -> !pageBlock.isEmpty()).collect(Collectors.toList());
+        return sectionBlocks.stream()
+                .filter(pageBlock -> !pageBlock.isEmpty())
+                .collect(Collectors.toList());
     }
 
+
     @Override
     public String toString() {
 
         return "OutlineObjectTreeNode{" + "textPageBlock=" + headline + '}';
     }
 
-
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Cell.java

@@ -35,14 +35,14 @@ public class Cell extends BoundingBox {
 
     public Cell(Point2D topLeft, Point2D bottomRight) {
 
-        this.bBoxInitialUserSpace = new Rectangle2D.Double(topLeft.getX(), topLeft.getY(), (bottomRight.getX() - topLeft.getX()), (bottomRight.getY() - topLeft.getY()));
-        this.bBox = bBoxInitialUserSpace;
+        this.bBoxPdf = new Rectangle2D.Double(topLeft.getX(), topLeft.getY(), (bottomRight.getX() - topLeft.getX()), (bottomRight.getY() - topLeft.getY()));
+        this.bBox = bBoxPdf;
     }
 
 
     public Cell(Rectangle2D bBoxInitialUserSpace, AffineTransform initialUserSpaceToJava) {
 
-        this.bBoxInitialUserSpace = bBoxInitialUserSpace;
+        this.bBoxPdf = bBoxInitialUserSpace;
         this.bBox = initialUserSpaceToJava.createTransformedShape(bBoxInitialUserSpace).getBounds2D();
     }
 
@@ -50,7 +50,7 @@ public class Cell extends BoundingBox {
     public static Cell copy(Cell cell) {
 
         Cell copy = new Cell();
-        copy.bBoxInitialUserSpace = cell.bBoxInitialUserSpace;
+        copy.bBoxPdf = cell.bBoxPdf;
         copy.bBox = cell.bBox;
         return copy;
     }
@@ -87,7 +87,7 @@ public class Cell extends BoundingBox {
 
         }
 
-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
+        return TextNormalizationUtilities.cleanString(sb.toString());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/CleanRulings.java

@@ -70,7 +70,7 @@ public class CleanRulings {
 
     public boolean lineBetween(BoundingBox a, BoundingBox b) {
 
-        return lineBetween(a.getBBoxInitialUserSpace(), b.getBBoxInitialUserSpace());
+        return lineBetween(a.getBBoxPdf(), b.getBBoxPdf());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/TablePageBlock.java

@@ -263,8 +263,8 @@ public class TablePageBlock extends AbstractPageBlock {
 
                         cells.stream()
                                 .map(originalCell -> new CellWithIntersection(originalCell,
-                                                                              RectangleTransformations.calculateIntersectedArea(cellFromGridStructure.getBBoxInitialUserSpace(),
-                                                                                                                                originalCell.getBBoxInitialUserSpace())))
+                                                                              RectangleTransformations.calculateIntersectedArea(cellFromGridStructure.getBBoxPdf(),
+                                                                                                                                originalCell.getBBoxPdf())))
                                 .filter(cellWithIntersection -> cellWithIntersection.intersectedArea > 0)
                                 .filter(cellWithIntersection -> cellWithIntersection.originalCell.getArea() > cellWithIntersection.intersectedArea * CELL_AREA_CONTAINED_THRESHOLD)
                                 .max(Comparator.comparing(CellWithIntersection::intersectedArea))

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/FontStyle.java

@@ -0,0 +1,9 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
+
+
+public enum FontStyle {
+    REGULAR,
+    BOLD,
+    ITALIC,
+    BOLD_ITALIC;
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -5,64 +5,50 @@ import java.awt.geom.Rectangle2D;
 
 import org.apache.pdfbox.text.TextPosition;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 
+import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
 
 @Data
-@Builder
+@SuperBuilder
 @NoArgsConstructor
 @AllArgsConstructor
-public class RedTextPosition extends BoundingBox {
+@EqualsAndHashCode(callSuper = true)
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class RedTextPosition extends TextBoundingBox {
 
     public final static int HEIGHT_PADDING = 2;
 
-    private Rectangle2D.Float bBoxDirAdj; // adjusted to text rotation
+    String unicode;
 
-    @JsonIgnore
-    private int rotation;
+    // estimated using the TextMatrix in radians
+    float exactDir;
 
-    @JsonIgnore
-    private float pageHeight;
+    float widthOfSpace;
 
-    @JsonIgnore
-    private float pageWidth;
+    float fontSizeInPt;
 
-    private String unicode;
-
-    @JsonIgnore
-    private float dir;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private float widthOfSpace;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private float fontSizeInPt;
-
-    // not used in reanalysis
-    @JsonIgnore
-    private String fontName;
+    String fontName;
 
 
     @SneakyThrows
     public static RedTextPosition fromTextPosition(TextPosition textPosition) {
 
         var pos = new RedTextPosition();
-        pos.setRotation(textPosition.getRotation());
-        pos.setPageHeight(textPosition.getPageHeight());
-        pos.setPageWidth(textPosition.getPageWidth());
         pos.setUnicode(textPosition.getUnicode());
-        pos.setDir(textPosition.getDir());
         pos.setWidthOfSpace(textPosition.getWidthOfSpace());
         pos.setFontSizeInPt(textPosition.getFontSizeInPt());
         pos.setFontName(textPosition.getFont().getName());
+        pos.setExactDir((float) FastAtan2.fastAtan2(textPosition.getTextMatrix().getShearY(), textPosition.getTextMatrix().getScaleX()));
+        pos.setDir(TextDirection.fromDegrees(textPosition.getDir()));
 
         //TODO: There is a mismatch in the java coords of the text and the rulings,
         // I guess if we start with the initial user space positions and transform them the same way we do the rulings it would work.
@@ -73,18 +59,18 @@ public class RedTextPosition extends BoundingBox {
                                                                  textPosition.getYDirAdj() - textHeight,
                                                                  textPosition.getWidthDirAdj(),
                                                                  textHeight + HEIGHT_PADDING);
+
         pos.setBBoxDirAdj(dirAdjPosition);
 
         AffineTransform affineTransform = getRotationMatrix(TextDirection.fromDegrees(textPosition.getDir()), textPosition.getPageWidth(), textPosition.getPageHeight());
         Rectangle2D bBoxInitialUserSpace = affineTransform.createTransformedShape(dirAdjPosition).getBounds2D();
 
-        pos.setBBoxInitialUserSpace(bBoxInitialUserSpace); // These are definitely correct
+        pos.setBBoxPdf(bBoxInitialUserSpace); // These are definitely correct
 
         return pos;
     }
 
 
-
     private static AffineTransform getRotationMatrix(TextDirection textDirection, float pageWidth, float pageHeight) {
 
         AffineTransform transform = new AffineTransform();
@@ -103,32 +89,4 @@ public class RedTextPosition extends BoundingBox {
         return transform;
     }
 
-
-    @JsonIgnore
-    public float getXDirAdj() {
-
-        return this.bBoxDirAdj.x;
-    }
-
-
-    @JsonIgnore
-    public float getYDirAdj() {
-
-        return this.bBoxDirAdj.y;
-    }
-
-
-    @JsonIgnore
-    public float getWidthDirAdj() {
-
-        return this.bBoxDirAdj.width;
-    }
-
-
-    @JsonIgnore
-    public float getHeightDir() {
-
-        return this.bBoxDirAdj.height;
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/SearchableText.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import java.util.ArrayList;
 import java.util.List;
+
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Getter;
@@ -38,11 +39,7 @@ public class SearchableText {
             sb.append(word);
             sb.append(' ');
         }
-        String text = sb.toString();
-        text = TextNormalizationUtilities.removeHyphenLineBreaks(text);
-        text = TextNormalizationUtilities.removeLineBreaks(text);
-        text = TextNormalizationUtilities.removeRepeatingWhitespaces(text);
-        return text;
+        return TextNormalizationUtilities.cleanString(sb.toString());
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextDirection.java

@@ -44,4 +44,15 @@ public enum TextDirection {
 
         throw new IllegalArgumentException(String.format("A value of %f is not supported by TextDirection", degrees));
     }
+
+
+    public int getRotation() {
+
+        return switch (this) {
+            case ZERO -> 0;
+            case QUARTER_CIRCLE -> 1;
+            case HALF_CIRCLE -> 2;
+            case THREE_QUARTER_CIRCLE -> 3;
+        };
+    }
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java

@@ -2,12 +2,14 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.AllArgsConstructor;
@@ -26,59 +28,62 @@ public class TextPageBlock extends AbstractPageBlock {
     @Builder.Default
     private List<TextPositionSequence> sequences = new ArrayList<>();
 
+    private Rectangle2D bBoxDirAdj;
+
     private String mostPopularWordFont;
 
     private String mostPopularWordStyle;
 
-    private float mostPopularWordFontSize;
+    private double mostPopularWordFontSize;
 
-    private float mostPopularWordHeight;
+    private double mostPopularWordHeight;
 
-    private float mostPopularWordSpaceWidth;
+    private double mostPopularWordSpaceWidth;
 
-    private float highestFontSize;
+    private double highestFontSize;
 
     private PageBlockType classification;
 
     private boolean toDuplicate;
 
+    private String text;
+    private boolean changed;
+
 
     public TextPageBlock(List<TextPositionSequence> sequences) {
 
-        this.sequences = sequences;
-        calculateFrequencyCounters();
+        this.sequences = new ArrayList<>(sequences);
+        if (!sequences.isEmpty()) {
+            calculateFrequencyCounters();
+        }
         calculateBBox();
     }
 
 
-    @JsonIgnore
+    public List<TextPositionSequence> getSequences() {
+
+        return Collections.unmodifiableList(sequences);
+    }
+
+
     public TextDirection getDir() {
 
         return sequences.get(0).getDir();
     }
 
 
-    @JsonIgnore
-    public float getPageHeight() {
-
-        return sequences.get(0).getPageHeight();
-    }
-
-
-    @JsonIgnore
-    public float getPageWidth() {
-
-        return sequences.get(0).getPageWidth();
-    }
-
-
     private void calculateBBox() {
 
         if (sequences == null) {
             this.bBox = new Rectangle2D.Double();
-            this.bBoxInitialUserSpace = new Rectangle2D.Double();
+            this.bBoxPdf = new Rectangle2D.Double();
+            this.bBoxDirAdj = new Rectangle2D.Double();
             return;
         }
+        this.bBoxDirAdj = sequences.stream()
+                .map(TextPositionSequence::getBBoxDirAdj)
+                .collect(RectangleTransformations.collectBBox());
+
         setToBBoxOfComponents(sequences);
     }
 
@@ -141,7 +146,7 @@ public class TextPageBlock extends AbstractPageBlock {
     public TextPageBlock union(TextPositionSequence r) {
 
         TextPageBlock union = this.copy();
-        union.getSequences().add(r);
+        union.add(r);
         calculateFrequencyCounters();
         calculateBBox();
         return union;
@@ -151,24 +156,35 @@ public class TextPageBlock extends AbstractPageBlock {
     public TextPageBlock union(TextPageBlock r) {
 
         TextPageBlock union = this.copy();
-        union.getSequences().addAll(r.getSequences());
+        union.addAll(r.getSequences());
         calculateFrequencyCounters();
         calculateBBox();
         return union;
     }
 
 
-    public void add(TextPageBlock r) {
+    public void add(TextPageBlock textPageBlock) {
 
-        sequences.addAll(r.getSequences());
+        changed = true;
+        sequences.addAll(textPageBlock.getSequences());
         calculateFrequencyCounters();
         calculateBBox();
     }
 
 
-    public void add(TextPositionSequence r) {
+    public void add(TextPositionSequence textPositionSequence) {
 
-        sequences.add(r);
+        changed = true;
+        sequences.add(textPositionSequence);
+        calculateFrequencyCounters();
+        calculateBBox();
+    }
+
+
+    public void addAll(List<TextPositionSequence> textPositionSequences) {
+
+        changed = true;
+        sequences.addAll(textPositionSequences);
         calculateFrequencyCounters();
         calculateBBox();
     }
@@ -203,22 +219,28 @@ public class TextPageBlock extends AbstractPageBlock {
     @JsonIgnore
     public String getText() {
 
-        StringBuilder sb = new StringBuilder();
+        if (text == null || changed) {
 
-        TextPositionSequence previous = null;
-        for (TextPositionSequence word : sequences) {
-            if (previous != null) {
-                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
-                    sb.append('\n');
-                } else {
-                    sb.append(' ');
+            StringBuilder sb = new StringBuilder();
+
+            TextPositionSequence previous = null;
+            for (TextPositionSequence word : sequences) {
+                if (previous != null) {
+                    if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
+                        sb.append('\n');
+                    } else {
+                        sb.append(' ');
+                    }
                 }
+                sb.append(word.toString());
+                previous = word;
             }
-            sb.append(word.toString());
-            previous = word;
+
+            text = TextNormalizationUtilities.removeHyphenLinebreaks(sb.toString());
+            changed = false;
         }
 
-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString());
+        return text;
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -1,20 +1,20 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
-import java.awt.geom.Rectangle2D;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition.HEIGHT_PADDING;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 import org.apache.pdfbox.text.TextPosition;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -23,27 +23,24 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
-public class TextPositionSequence extends BoundingBox implements CharSequence {
+public class TextPositionSequence extends TextBoundingBox implements CharSequence {
 
-    public static final int HEIGHT_PADDING = 2;
+    public static final String STANDARD = "standard";
+    public static final String BOLD_ITALIC = "bold, italic";
+    public static final String BOLD = "bold";
+    public static final String ITALIC = "italic";
 
-    @EqualsAndHashCode.Include
     private int page;
 
-    @EqualsAndHashCode.Include
+    @Builder.Default
     private List<RedTextPosition> textPositions = new ArrayList<>();
 
-    private Rectangle2D bBoxDirAdj;
-    @EqualsAndHashCode.Include
-    private TextDirection dir;
-    private int rotation;
-    private float pageHeight;
-    private float pageWidth;
     private boolean isParagraphStart;
     private boolean strikethrough;
     private boolean underline;
 
+    private Integer hashcodeCache;
+
 
     public TextPositionSequence(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
 
@@ -51,21 +48,15 @@ public class TextPositionSequence extends BoundingBox implements CharSequence {
                 .map(RedTextPosition::fromTextPosition)
                 .collect(Collectors.toList());
         this.page = pageNumber;
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
         this.isParagraphStart = isParagraphStart;
-        calculateBBox();
+        calculateBBoxAndHashcode();
     }
 
 
-    private void calculateBBox() {
+    private void calculateBBoxAndHashcode() {
 
-        this.bBoxDirAdj = textPositions.stream()
-                .map(RedTextPosition::getBBoxDirAdj)
-                .collect(RectangleTransformations.collectBBox());
         setToBBoxOfComponents(getTextPositions());
+        hashcodeCache = null;
     }
 
 
@@ -73,11 +64,7 @@ public class TextPositionSequence extends BoundingBox implements CharSequence {
 
         this.textPositions = textPositions;
         this.page = page;
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        calculateBBox();
+        calculateBBoxAndHashcode();
     }
 
 
@@ -112,9 +99,6 @@ public class TextPositionSequence extends BoundingBox implements CharSequence {
         textPositionSequence.textPositions = textPositions.subList(start, end);
         textPositionSequence.page = page;
         textPositionSequence.dir = dir;
-        textPositionSequence.rotation = rotation;
-        textPositionSequence.pageHeight = pageHeight;
-        textPositionSequence.pageWidth = pageWidth;
         textPositionSequence.setToBBoxOfComponents(getTextPositions());
         return textPositionSequence;
     }
@@ -141,90 +125,26 @@ public class TextPositionSequence extends BoundingBox implements CharSequence {
 
         this.textPositions.add(textPosition);
         this.page = textPositionSequence.getPage();
-        this.dir = textPositionSequence.getDir();
-        this.rotation = textPositionSequence.getRotation();
-        this.pageHeight = textPositionSequence.getPageHeight();
-        this.pageWidth = textPositionSequence.getPageWidth();
-        calculateBBox();
+        calculateBBoxAndHashcode();
     }
 
 
     public void add(TextPosition textPosition) {
 
         this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        calculateBBox();
+        calculateBBoxAndHashcode();
     }
 
 
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted minX value
-     */
-
-    public float getMinXDirAdj() {
-
-        return textPositions.get(0).getXDirAdj();
+    public double getTextHeightNoPadding() {
 
+        return textPositions.get(0).getHeightDirAdj();
     }
 
 
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted maxX value
-     */
+    public double getTextHeight() {
 
-    public float getMaxXDirAdj() {
-
-        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
-
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
-     */
-
-    public float getMinYDirAdj() {
-
-        return textPositions.get(0).getYDirAdj() - getTextHeight();
-
-    }
-
-
-    /**
-     * This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
-     * This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
-     *
-     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
-     */
-
-    public float getMaxYDirAdj() {
-
-        return textPositions.get(0).getYDirAdj();
-
-    }
-
-
-    public float getTextHeightNoPadding() {
-
-        return textPositions.get(0).getHeightDir();
-    }
-
-
-    public float getTextHeight() {
-
-        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
+        return textPositions.get(0).getHeightDirAdj() + HEIGHT_PADDING;
     }
 
 
@@ -240,18 +160,18 @@ public class TextPositionSequence extends BoundingBox implements CharSequence {
     public String getFontStyle() {
 
         if (textPositions.get(0).getFontName() == null) {
-            return "standard";
+            return STANDARD;
         }
         String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);
 
-        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
-            return "bold, italic";
-        } else if (lowercaseFontName.contains("bold")) {
-            return "bold";
-        } else if (lowercaseFontName.contains("italic")) {
-            return "italic";
+        if (lowercaseFontName.contains(BOLD) && lowercaseFontName.contains(ITALIC)) {
+            return BOLD_ITALIC;
+        } else if (lowercaseFontName.contains(BOLD)) {
+            return BOLD;
+        } else if (lowercaseFontName.contains(ITALIC)) {
+            return ITALIC;
         } else {
-            return "standard";
+            return STANDARD;
         }
     }
 
@@ -267,5 +187,55 @@ public class TextPositionSequence extends BoundingBox implements CharSequence {
         return textPositions.get(0).getWidthOfSpace();
     }
 
+
+    public boolean equals(final Object o) {
+        // auto-generated with lombok
+        if (o == this) {
+            return true;
+        }
+        if (!(o instanceof TextPositionSequence other)) {
+            return false;
+        }
+        if (!other.canEqual((Object) this)) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        if (this.getPage() != other.getPage()) {
+            return false;
+        }
+        final Object this$textPositions = this.getTextPositions();
+        final Object other$textPositions = other.getTextPositions();
+        if (!Objects.equals(this$textPositions, other$textPositions)) {
+            return false;
+        }
+        return Objects.equals(this.getHashcodeCache(), other.getHashcodeCache());
+    }
+
+
+    protected boolean canEqual(final Object other) {return other instanceof TextPositionSequence;}
+
+
+    public int hashCode() {
+
+        if (hashcodeCache == null) {
+            hashcodeCache = hashcodeCalculation();
+        }
+
+        return hashcodeCache;
+    }
+
+
+    private int hashcodeCalculation() {
+
+        final int PRIME = 59;
+        int result = super.hashCode();
+        result = result * PRIME + this.getPage();
+        final Object $textPositions = this.getTextPositions();
+        result = result * PRIME + ($textPositions == null ? 43 : $textPositions.hashCode());
+        return result;
+    }
+
 }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/ImageServiceResponseAdapter.java

@@ -56,7 +56,7 @@ public class ImageServiceResponseAdapter {
         classificationPage.getImages().forEach(image -> {
             if (image.getImageType().equals(ImageType.OTHER)) {
                 for (AbstractPageBlock textblock : classificationPage.getTextBlocks()) {
-                    if (image.getPosition().contains(textblock.getBBoxInitialUserSpace())) {
+                    if (image.getPosition().contains(textblock.getBBoxPdf())) {
                         image.setImageType(ImageType.OCR);
                         return;
                     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Classification.java

@@ -14,6 +14,7 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class Classification {
 
+    @Builder.Default
     private Map<String, Float> probabilities = new HashMap<>();
     private String label;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageServiceResponse.java

@@ -22,8 +22,10 @@ public class ImageServiceResponse {
 
     @JsonProperty(value = "imageMetadata")
     @JsonAlias("data")
+    @Builder.Default
     private List<ImageMetadata> data = new ArrayList<>();
 
+    @Builder.Default
     private List<ImageMetadata> dataCV = new ArrayList<>();
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableData.java

@@ -15,6 +15,7 @@ import lombok.NoArgsConstructor;
 public class TableData {
 
     private PageInfo pageInfo;
+    @Builder.Default
     private List<TableCells> tableCells = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableServiceResponse.java

@@ -19,7 +19,7 @@ public class TableServiceResponse {
     private String operation;
     private String targetFileExtension;
     private String responseFileExtension;
-
+    @Builder.Default
     private List<TableData> data = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/GapDetectionService.java

@@ -6,7 +6,6 @@ import java.util.List;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.AllArgsConstructor;
 import lombok.experimental.UtilityClass;
@@ -38,7 +37,7 @@ public class GapDetectionService {
         for (TextPositionSequence currentTextPosition : sortedTextPositionSequences.subList(1, sortedTextPositionSequences.size())) {
 
             double yDifference = Math.abs(currentTextPosition.getMaxYDirAdj() - previousTextPosition.getMaxYDirAdj());
-            double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getMinXDirAdj());
+            double xGap = Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj());
             Rectangle2D previousTextPositionBBox = toRectangle2D(previousTextPosition);
             Rectangle2D currentTextPositionBBox = toRectangle2D(currentTextPosition);
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/LineDetectionService.java

@@ -71,7 +71,7 @@ public class LineDetectionService {
 
     private static boolean isXGap(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
 
-        return Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getMinXDirAdj()) > (avgTextPositionHeight * X_GAP_FACTOR);
+        return Math.abs(previousTextPosition.getMaxXDirAdj() - currentTextPosition.getXDirAdj()) > (avgTextPositionHeight * X_GAP_FACTOR);
     }
 
 
@@ -83,7 +83,7 @@ public class LineDetectionService {
 
     private static boolean isNewLine(TextPositionSequence currentTextPosition, TextPositionSequence previousTextPosition, double avgTextPositionHeight) {
 
-        return Math.abs(previousTextPosition.getMinYDirAdj() - currentTextPosition.getMinYDirAdj()) > avgTextPositionHeight;
+        return Math.abs(previousTextPosition.getYDirAdj() - currentTextPosition.getYDirAdj()) > avgTextPositionHeight;
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -34,7 +34,22 @@ public class SimplifiedSectionTextService {
         List<SimplifiedSectionText> simplifiedText = Stream.of(simplifiedMainSectionsList, simplifiedHeadersList, simplifiedFootersList)
                 .flatMap(List::stream)
                 .collect(Collectors.toList());
-        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedText).build();
+        return SimplifiedText.builder()
+                .numberOfPages(document.getNumberOfPages())
+                .sectionTexts(simplifiedText)
+                .mainSectionNumbers(document.getAllSections()
+                                            .stream()
+                                            .map(this::getSectionNumber)
+                                            .toList())
+                .headerSectionNumbers(document.getHeaders()
+                                              .stream()
+                                              .map(this::getSectionNumber)
+                                              .toList())
+                .footerSectionNumbers(document.getFooters()
+                                              .stream()
+                                              .map(this::getSectionNumber)
+                                              .toList())
+                .build();
     }
 
 
@@ -49,4 +64,13 @@ public class SimplifiedSectionTextService {
                 .build();
     }
 
+
+    private String getSectionNumber(SemanticNode semanticNode) {
+
+        return semanticNode.getTreeId()
+                .stream()
+                .map(String::valueOf)
+                .collect(Collectors.joining("."));
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/TableExtractionService.java

@@ -78,7 +78,7 @@ public class TableExtractionService {
 
             List<Cell> containedCells = new ArrayList<>();
             for (Cell c : cells) {
-                if (c.hasMinimumSize() && area.contains(c.getBBoxInitialUserSpace())) {
+                if (c.hasMinimumSize() && area.contains(c.getBBoxPdf())) {
                     containedCells.add(c);
                 }
             }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/TextRulingsClassifier.java

@@ -31,13 +31,13 @@ public class TextRulingsClassifier {
 
     private static void handleVerticalText(CleanRulings cleanRulings, TextPositionSequence word) {
 
-        float lowerY = (float) (word.getBBoxInitialUserSpace().getMinY() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
-        float upperY = (float) (word.getBBoxInitialUserSpace().getMaxY() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
+        float lowerY = (float) (word.getBBoxPdf().getMinY() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
+        float upperY = (float) (word.getBBoxPdf().getMaxY() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
 
-        float strikethroughCenterX = (float) word.getBBoxInitialUserSpace().getCenterX();
+        float strikethroughCenterX = (float) word.getBBoxPdf().getCenterX();
         float strikethroughBoxHeight = (float) ((word.getBBoxDirAdj().getHeight() * STRIKETHROUGH_ZONE) / 2);
 
-        float underlineCenterX = (float) (word.getDir().equals(TextDirection.QUARTER_CIRCLE) ? word.getBBoxInitialUserSpace().getMaxX() : word.getBBoxInitialUserSpace().getMinX());
+        float underlineCenterX = (float) (word.getDir().equals(TextDirection.QUARTER_CIRCLE) ? word.getBBoxPdf().getMaxX() : word.getBBoxPdf().getMinX());
         float underlineBoxHeight = (float) ((word.getBBoxDirAdj().getHeight() * UNDERLINE_ZONE) / 2);
 
         float leftX = Math.min(underlineCenterX - underlineBoxHeight, strikethroughCenterX - strikethroughBoxHeight);
@@ -65,13 +65,13 @@ public class TextRulingsClassifier {
 
     private static void handleHorizontalText(CleanRulings cleanRulings, TextPositionSequence word) {
 
-        float leftX = (float) (word.getBBoxInitialUserSpace().getMinX() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
-        float rightX = (float) (word.getBBoxInitialUserSpace().getMaxX() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
+        float leftX = (float) (word.getBBoxPdf().getMinX() + TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
+        float rightX = (float) (word.getBBoxPdf().getMaxX() - TEXT_BBOX_THRESHOLD_FACTOR * word.getWidth());
 
-        float strikethroughCenterY = (float) word.getBBoxInitialUserSpace().getCenterY();
+        float strikethroughCenterY = (float) word.getBBoxPdf().getCenterY();
         float strikethroughBoxHeight = (float) ((word.getBBoxDirAdj().getHeight() * STRIKETHROUGH_ZONE) / 2);
 
-        float underlineCenterY = (float) (word.getDir().equals(TextDirection.ZERO) ? word.getBBoxInitialUserSpace().getMinY() : word.getBBoxInitialUserSpace().getMaxY());
+        float underlineCenterY = (float) (word.getDir().equals(TextDirection.ZERO) ? word.getBBoxPdf().getMinY() : word.getBBoxPdf().getMaxY());
         float underlineBoxHeight = (float) ((word.getBBoxDirAdj().getHeight() * UNDERLINE_ZONE) / 2);
 
         float lowerY = Math.min(underlineCenterY - underlineBoxHeight, strikethroughCenterY - strikethroughBoxHeight);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/BlockificationPostprocessingService.java

@@ -2,14 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor.services.blockifica
 
 import static com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService.buildTextBlock;
 
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Locale;
-import java.util.function.Function;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
@@ -19,23 +18,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentif
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.Data;
 
 @Service
 public class BlockificationPostprocessingService {
 
-    private static final float BLOCK_TO_OUTLINE_DISTANCE_THRESHOLD = 5.0f;
-
-    private static final Function<TextPageBlock, Rectangle2D> blockToBoundingBox = (abstractPageBlock) -> abstractPageBlock.getSequences()
-            .stream()
-            .map(textPositionSequence -> textPositionSequence.getTextPositions()
-                    .stream()
-                    .map(tp -> SearchTextWithTextPositionFactory.mapRedTextPositionToInitialUserSpace(tp, textPositionSequence))
-                    .collect(RectangleTransformations.collectBBox()))
-            .collect(RectangleTransformations.collectBBox());
+    private static final float STRING_SIMILARITY_THRESHOLD = 0.1f;
 
 
     public OutlineObject sanitizeOutlineBlocks(ClassificationPage classificationPage, OutlineObject notFoundOutlineObject) {
@@ -46,20 +35,18 @@ public class BlockificationPostprocessingService {
             return null;
         }
 
-        float pageHeight = classificationPage.getPageHeight();
-
         ListIterator<OutlineObject> outlineObjectListIterator = outlineObjects.listIterator();
 
         if (notFoundOutlineObject != null) {
             OutlineProcessionContext notFoundOutlineObjectProcessionContext = new OutlineProcessionContext(notFoundOutlineObject);
-            processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, notFoundOutlineObjectProcessionContext);
+            processTextBlocks(getTextPageBlocks(classificationPage), notFoundOutlineObjectProcessionContext);
 
             OutlineObject firstOutlineObject = null;
             OutlineProcessionContext firstOutlineObjectProcessionContext = null;
             if (outlineObjectListIterator.hasNext()) {
                 firstOutlineObject = outlineObjectListIterator.next();
                 firstOutlineObjectProcessionContext = new OutlineProcessionContext(firstOutlineObject);
-                processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, firstOutlineObjectProcessionContext);
+                processTextBlocks(getTextPageBlocks(classificationPage), firstOutlineObjectProcessionContext);
             }
 
             if (!contextsOverlap(notFoundOutlineObjectProcessionContext, firstOutlineObjectProcessionContext)) {
@@ -68,7 +55,7 @@ public class BlockificationPostprocessingService {
             if (firstOutlineObject != null) {
                 // re-create the context for the updated blocks
                 firstOutlineObjectProcessionContext = new OutlineProcessionContext(firstOutlineObject);
-                processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, firstOutlineObjectProcessionContext);
+                processTextBlocks(getTextPageBlocks(classificationPage), firstOutlineObjectProcessionContext);
                 firstOutlineObject.setFound(selectMatch(classificationPage, firstOutlineObjectProcessionContext));
             }
 
@@ -76,7 +63,7 @@ public class BlockificationPostprocessingService {
 
         outlineObjectListIterator.forEachRemaining(outlineObject -> {
             OutlineProcessionContext outlineObjectProcessionContext = new OutlineProcessionContext(outlineObject);
-            processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, outlineObjectProcessionContext);
+            processTextBlocks(getTextPageBlocks(classificationPage), outlineObjectProcessionContext);
             outlineObject.setFound(selectMatch(classificationPage, outlineObjectProcessionContext));
         });
 
@@ -116,8 +103,7 @@ public class BlockificationPostprocessingService {
 
         double maxYFirst = blocksOfFirstOutline.stream()
                 .mapToDouble(TextPageBlock::getPdfMaxY)
-                .max()
-                .orElse(Double.NEGATIVE_INFINITY);
+                .max().orElse(Double.NEGATIVE_INFINITY);
 
         return blocksOfNotFoundOutline.stream()
                 .mapToDouble(TextPageBlock::getPdfMaxY)
@@ -139,13 +125,13 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private void processTextBlocks(List<TextPageBlock> textBlocks, float pageHeight, OutlineProcessionContext context) {
+    private void processTextBlocks(List<TextPageBlock> textBlocks, OutlineProcessionContext context) {
 
         OutlineObject outlineObject = context.getOutlineObject();
         ListIterator<TextPageBlock> iterator = textBlocks.listIterator();
         while (iterator.hasNext()) {
             TextPageBlock pageBlock = iterator.next();
-            if (pageHeight - outlineObject.getPoint().getY() - BLOCK_TO_OUTLINE_DISTANCE_THRESHOLD <= pageBlock.getMaxY()) {
+            if (outlineObject.isAbove(pageBlock)) {
                 break;
             }
         }
@@ -190,8 +176,7 @@ public class BlockificationPostprocessingService {
             for (List<TextPageBlock> combination : combinations) {
                 double averageDistance = combination.stream()
                         .map(block -> calculateDistance(outlineObject, block))
-                        .mapToDouble(Double::doubleValue).average()
-                        .orElse(Double.MAX_VALUE);
+                        .mapToDouble(Double::doubleValue).average().orElse(Double.MAX_VALUE);
                 if (distanceToBestMergeCandidates > averageDistance) {
                     distanceToBestMergeCandidates = averageDistance;
                     bestMergeCandidateCombination = combination;
@@ -273,6 +258,10 @@ public class BlockificationPostprocessingService {
         List<TextPositionSequence> postSequence = new ArrayList<>();
         StringBuilder currentSequence = new StringBuilder();
 
+        if (target.isBlank()) {
+            return new WordSequenceResult();
+        }
+
         for (TextPositionSequence sequence : textPositionSequences) {
 
             currentSequence.append(sanitizeString(sequence.toString()));
@@ -332,7 +321,7 @@ public class BlockificationPostprocessingService {
         TextPositionSequence in = null;
         TextPositionSequence out;
 
-        String currentSequence = sequence.toString();
+        String currentSequence = sequence.toString().toLowerCase(Locale.ROOT);
         int index = currentSequence.indexOf(toRemove);
         int endIndex = index + toRemove.length();
 
@@ -368,7 +357,7 @@ public class BlockificationPostprocessingService {
                 if (firstBlock != null && !firstBlock.getSequences().isEmpty()) {
 
                     if (textPageBlock.getDir() == firstBlock.getDir()) {
-                        firstBlock.getSequences().addAll(textPageBlock.getSequences());
+                        firstBlock.addAll(textPageBlock.getSequences());
                         mergedBlocks.add(textPageBlock);
                     }
                 }
@@ -416,9 +405,7 @@ public class BlockificationPostprocessingService {
 
     private double calculateDistance(OutlineObject outlineObject, TextPageBlock pageBlock) {
 
-        double deltaX = outlineObject.getPoint().getX() - pageBlock.getMinX();
-        double deltaY = pageBlock.getPageHeight() - outlineObject.getPoint().getY() - pageBlock.getMinY();
-        return Math.sqrt(deltaX * deltaX + deltaY * deltaY);
+        return outlineObject.distance(pageBlock);
     }
 
 
@@ -435,6 +422,13 @@ public class BlockificationPostprocessingService {
         String blockText = sanitizeString(pageBlock.getText());
         String outlineTitle = sanitizeString(outlineObject.getTitle());
 
+        int threshold = (int) (Math.min(blockText.length(), outlineTitle.length()) * STRING_SIMILARITY_THRESHOLD) + 1;
+        int distance = new LevenshteinDistance(threshold).apply(blockText, outlineTitle);
+        if (distance >= 0 && distance < threshold) {
+            context.directMatch = pageBlock;
+            return true;
+        }
+
         boolean blockTextContainsOutlineTitle = blockText.contains(outlineTitle);
         boolean outlineTitleContainsBlockText = outlineTitle.contains(blockText);
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;
 
 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
 import java.util.ListIterator;
 
@@ -10,7 +9,6 @@ import org.springframework.stereotype.Service;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
@@ -40,7 +38,7 @@ public class DocstrumBlockificationService {
 
         CleanRulings usedRulings = rulings.withoutTextRulings();
 
-        var zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder, usedRulings, visualizations);
+        List<Zone> zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder, usedRulings, visualizations);
 
         if (!textPositions.isEmpty()) {
             visualizations.addZoneVisualizations(zones, textPositions.get(0).getPage());
@@ -48,11 +46,7 @@ public class DocstrumBlockificationService {
             visualizations.addCharactersWithNeighbours(zones, textPositions.get(0).getPage());
         }
 
-        var pageBlocks = toAbstractPageBlocks(zones, xyOrder, usedRulings);
-
-        if (xyOrder) {
-            sortPageBlocksXThenY(pageBlocks);
-        }
+        var pageBlocks = toAbstractPageBlocks(zones);
 
         var classificationPage = new ClassificationPage(pageBlocks);
         classificationPage.setCleanRulings(rulings);
@@ -73,21 +67,7 @@ public class DocstrumBlockificationService {
     }
 
 
-    private static void sortPageBlocksXThenY(List<AbstractPageBlock> pageBlocks) {
-
-        pageBlocks.sort(Comparator.comparing(AbstractPageBlock::getMinY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                                .thenComparing(AbstractPageBlock::getMinX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-        pageBlocks.sort(new Comparator<AbstractPageBlock>() {
-            @Override
-            public int compare(AbstractPageBlock o1, AbstractPageBlock o2) {
-
-                return Math.abs(o1.getMinY() - o2.getMinY()) < 5 && o1.getMinX() < o2.getMinX() == true ? -1 : 0;
-            }
-        });
-    }
-
-
-    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones, boolean xyOrder, CleanRulings usedRulings) {
+    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones) {
 
         List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
         zones.forEach(zone -> {
@@ -202,7 +182,7 @@ public class DocstrumBlockificationService {
 
     private TextPageBlock combineBlocksAndResetIterator(TextPageBlock previous, TextPageBlock current, ListIterator<AbstractPageBlock> itty, boolean toDuplicate) {
 
-        previous.getSequences().addAll(current.getSequences());
+        previous.addAll(current.getSequences());
         previous = buildTextBlock(previous.getSequences(), 0);
         previous.setToDuplicate(toDuplicate);
         if (current.getClassification() != null && previous.getClassification() == null) {
@@ -296,10 +276,14 @@ public class DocstrumBlockificationService {
                     continue;
                 }
 
+//                if (!current.getMostPopularWordStyle().equals(inner.getMostPopularWordStyle())) {
+//                    continue;
+//                }
+
                 if (current.getDir() == inner.getDir() && current.intersects(inner, yThreshold, xThreshold)) {
 
                     boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
-                    current.getSequences().addAll(inner.getSequences());
+                    current.addAll(inner.getSequences());
                     current = buildTextBlock(current.getSequences(), 0);
 
                     current.setToDuplicate(toDuplicate);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocuMineBlockificationService.java

@@ -1,25 +1,24 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;
 
-import static java.util.stream.Collectors.toSet;
-
 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
+@SuppressWarnings("all")
 @Service
 public class DocuMineBlockificationService {
 
@@ -44,31 +43,32 @@ public class DocuMineBlockificationService {
 
         CleanRulings usedRulings = cleanRulings.withoutTextRulings();
 
-        float minX = 1000;
-        float maxX = 0;
-        float minY = 1000;
-        float maxY = 0;
+        double minX = 1000;
+        double maxX = 0;
+        double minY = 1000;
+        double maxY = 0;
         TextPositionSequence prev = null;
 
         boolean wasSplitted = false;
-        Float splitX1 = null;
+        Double splitX1 = null;
         for (TextPositionSequence word : textPositions) {
 
-            boolean lineSeparation = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * 1.25;
-            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
-            boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
-            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < -5;
-            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
+            boolean lineSeparation = prev != null && word.getYDirAdj() - prev.getMaxYDirAdj() > Math.min(word.getHeight(), prev.getHeight()) * 1.1;
+            boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
+            boolean splitByX = prev != null && maxX + 50 < word.getXDirAdj() && prev.getYDirAdj() == word.getYDirAdj();
+            boolean negativeXGap = prev != null && word.getXDirAdj() - minX < -5;
+            boolean newLineAfterSplit = prev != null && word.getYDirAdj() != prev.getYDirAdj() && wasSplitted && splitX1 != word.getXDirAdj();
             boolean isSplitByRuling = prev != null && usedRulings.lineBetween(prev, word);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 && (word.getFontStyle()
-                                                                                                                                                               .contains("bold")
-                                                                                                                                                       && !prev.getFontStyle()
-                    .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
+            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 //
+                                                && (word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold")
+                                                    || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold")
+                                                    || Math.abs(prev.getFontSize() - word.getFontSize()) >= 1
+                                                    || Math.abs(word.getTextHeight() - prev.getTextHeight()) > 0.8);
 
             Matcher matcher = pattern.matcher(chunkWords.stream()
                                                       .collect(Collectors.joining(" ")).toString());
-            boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();
+            boolean startsOnSameX = Math.abs(minX - word.getXDirAdj()) < 5 && matcher.matches();
 
             if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap || startsOnSameX)) {
 
@@ -84,7 +84,7 @@ public class DocuMineBlockificationService {
                 if (splitByX && !isSplitByRuling) {
                     wasSplitted = true;
                     cb1.setOrientation(Orientation.LEFT);
-                    splitX1 = word.getMinXDirAdj();
+                    splitX1 = word.getXDirAdj();
                 } else if (newLineAfterSplit && !isSplitByRuling) {
                     wasSplitted = false;
                     cb1.setOrientation(Orientation.RIGHT);
@@ -107,14 +107,14 @@ public class DocuMineBlockificationService {
             chunkWords.add(word);
 
             prev = word;
-            if (word.getMinXDirAdj() < minX) {
-                minX = word.getMinXDirAdj();
+            if (word.getXDirAdj() < minX) {
+                minX = word.getXDirAdj();
             }
             if (word.getMaxXDirAdj() > maxX) {
                 maxX = word.getMaxXDirAdj();
             }
-            if (word.getMinYDirAdj() < minY) {
-                minY = word.getMinYDirAdj();
+            if (word.getYDirAdj() < minY) {
+                minY = word.getYDirAdj();
             }
             if (word.getMaxYDirAdj() > maxY) {
                 maxY = word.getMaxYDirAdj();
@@ -127,6 +127,76 @@ public class DocuMineBlockificationService {
     }
 
 
+    public void mergeblocks(ClassificationPage page, CleanRulings usedRulings, float xThreshold, float yThreshold) {
+
+        var blocks = page.getTextBlocks();
+        ListIterator<AbstractPageBlock> itty = blocks.listIterator();
+        while (itty.hasNext()) {
+            AbstractPageBlock block = itty.next();
+            if (block == null) {
+                continue;
+            }
+            if (block instanceof TablePageBlock) {
+                continue;
+            }
+
+            TextPageBlock current = (TextPageBlock) block;
+
+            for (int i = 0; i < blocks.size(); i++) {
+
+                AbstractPageBlock abstractPageBlock = blocks.get(i);
+                if (abstractPageBlock == null) {
+                    continue;
+                }
+                if (abstractPageBlock == current) {
+                    continue;
+                }
+                if (abstractPageBlock instanceof TablePageBlock) {
+                    continue;
+                }
+
+                if (isHeadlineFromOutline(current) || isHeadlineFromOutline(abstractPageBlock)) {
+                    continue;
+                }
+
+                TextPageBlock inner = (TextPageBlock) abstractPageBlock;
+
+                if (usedRulings.lineBetween(current, blocks.get(i))) {
+                    continue;
+                }
+
+                if (current.getDir() == inner.getDir() && current.intersects(inner, yThreshold, xThreshold) && (current.getClassification() == null || current.getClassification()
+                        .equals(inner.getClassification()))) {
+
+                    boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
+                    current.addAll(inner.getSequences());
+                    current = buildTextBlock(current.getSequences(), 0);
+                    current.setClassification(inner.getClassification());
+                    current.setToDuplicate(toDuplicate);
+                    blocks.set(i, null);
+                    itty.set(current);
+                }
+            }
+        }
+        var blocksIterator = blocks.iterator();
+        while (blocksIterator.hasNext()) {
+            if (blocksIterator.next() == null) {
+                blocksIterator.remove();
+            }
+        }
+    }
+
+
+    private boolean isHeadlineFromOutline(AbstractPageBlock abstractPageBlock) {
+
+        return abstractPageBlock.getEngines().contains(LayoutEngine.OUTLINE) && abstractPageBlock.getClassification() != null && abstractPageBlock.getClassification().isHeadline();
+    }
+
+
+    public static TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+
+        return new TextPageBlock(wordBlockList);
+    }
 
 }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java

@@ -38,18 +38,18 @@ public class RedactManagerBlockificationService {
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractPageBlock> chunkBlockList = new ArrayList<>();
 
-        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
+        double minX = 1000, maxX = 0, minY = 1000, maxY = 0;
         TextPositionSequence prev = null;
 
         boolean wasSplitted = false;
-        Float splitX1 = null;
+        Double splitX1 = null;
         for (TextPositionSequence word : textPositions) {
 
-            boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25;
-            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
-            boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
-            boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
-            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
+            boolean lineSeparation = word.getYDirAdj() - maxY > word.getHeight() * 1.25;
+            boolean startFromTop = prev != null && word.getYDirAdj() < prev.getYDirAdj() - prev.getTextHeight();
+            boolean splitByX = prev != null && maxX + 50 < word.getXDirAdj() && prev.getYDirAdj() == word.getYDirAdj();
+            boolean xIsBeforeFirstX = prev != null && word.getXDirAdj() < minX;
+            boolean newLineAfterSplit = prev != null && word.getYDirAdj() != prev.getYDirAdj() && wasSplitted && splitX1 != word.getXDirAdj();
             boolean isSplitByRuling = prev != null && usedRulings.lineBetween(prev, word);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
 
@@ -69,7 +69,7 @@ public class RedactManagerBlockificationService {
                 if (splitByX && !isSplitByRuling) {
                     wasSplitted = true;
                     cb1.setOrientation(Orientation.LEFT);
-                    splitX1 = word.getMinXDirAdj();
+                    splitX1 = word.getXDirAdj();
                 } else if (newLineAfterSplit && !isSplitByRuling) {
                     wasSplitted = false;
                     cb1.setOrientation(Orientation.RIGHT);
@@ -92,14 +92,14 @@ public class RedactManagerBlockificationService {
             chunkWords.add(word);
 
             prev = word;
-            if (word.getMinXDirAdj() < minX) {
-                minX = word.getMinXDirAdj();
+            if (word.getXDirAdj() < minX) {
+                minX = word.getXDirAdj();
             }
             if (word.getMaxXDirAdj() > maxX) {
                 maxX = word.getMaxXDirAdj();
             }
-            if (word.getMinYDirAdj() < minY) {
-                minY = word.getMinYDirAdj();
+            if (word.getYDirAdj() < minY) {
+                minY = word.getYDirAdj();
             }
             if (word.getMaxYDirAdj() > maxY) {
                 maxY = word.getMaxYDirAdj();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClarifyndClassificationService.java

@@ -23,7 +23,7 @@ public class ClarifyndClassificationService {
 
     public void classifyDocument(ClassificationDocument document) {
 
-        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
@@ -35,7 +35,10 @@ public class ClarifyndClassificationService {
     }
 
 
-    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService,
+                              ClassificationPage page,
+                              ClassificationDocument document,
+                              List<Double> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
@@ -45,7 +48,11 @@ public class ClarifyndClassificationService {
     }
 
 
-    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
+                               TextPageBlock textBlock,
+                               ClassificationPage page,
+                               ClassificationDocument document,
+                               List<Double> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
@@ -57,59 +64,58 @@ public class ClarifyndClassificationService {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
             return;
         }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
-                textBlock,
-                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+            || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
+                                                                                                   || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
                 .getMostPopular())) {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
 
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
-                textBlock,
-                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                   || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
+                                                                                                           || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
                 .getMostPopular())) {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
-        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
-                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
-                .size() == 1)) {
+        } else if (page.getPageNumber() == 1 //
+                   && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
+                       && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                 textBlock.setClassification(PageBlockType.TITLE);
             }
-        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                .getCountPerValue()
-                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()
+                   && PositionUtils.getApproxLineCount(textBlock) < 4.9
+                   && (textBlock.getMostPopularWordStyle().equals("bold")
+                       || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
+                          && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
+                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                    PageBlockType headlineType = PageBlockType.getHeadlineType(i);
-                    headlineClassificationService.classifyHeadline(textBlock, headlineType);
-                    document.setHeadlines(true);
-                }
-            }
-        } else if (!textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-            PageBlockType headlineType = PageBlockType.getHeadlineType(headlineFontSizes.size() + 1);
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
             document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+        } else if (!textBlock.getText().startsWith("Figure ")
+                   && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
+                   && textBlock.getMostPopularWordStyle().equals("bold")
+                   && !document.getFontStyleCounter().getMostPopular().equals("bold")
+                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
+                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+            headlineClassificationService.classifyHeadline(textBlock, headlineType);
+            document.setHeadlines(true);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
+                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
+                   && textBlock.getMostPopularWordStyle().equals("bold")
+                   && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
             textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
-                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
-                .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
+                   && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular())
+                   && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular())
+                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
             textBlock.setClassification(PageBlockType.PARAGRAPH);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
+                   && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
+                   && textBlock.getMostPopularWordStyle().equals("italic")
+                   && !document.getFontStyleCounter().getMostPopular().equals("italic")
+                   && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
             textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
             textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/DocuMineClassificationService.java

@@ -31,7 +31,7 @@ public class DocuMineClassificationService {
 
     public void classifyDocument(ClassificationDocument document) {
 
-        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
@@ -46,7 +46,7 @@ public class DocuMineClassificationService {
     private void classifyPage(HeadlineClassificationService headlineClassificationService,
                               ClassificationPage page,
                               ClassificationDocument document,
-                              List<Float> headlineFontSizes) {
+                              List<Double> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
@@ -60,7 +60,7 @@ public class DocuMineClassificationService {
                                TextPageBlock textBlock,
                                ClassificationPage page,
                                ClassificationDocument document,
-                               List<Float> headlineFontSizes) {
+                               List<Double> headlineFontSizes) {
 
         log.debug("headlineFontSizes: {}", headlineFontSizes);
         var bodyTextFrame = page.getBodyTextFrame();
@@ -74,7 +74,7 @@ public class DocuMineClassificationService {
             return;
         }
         if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
             return;
         }
         if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) //
@@ -108,22 +108,26 @@ public class DocuMineClassificationService {
                        && Character.isDigit(textBlock.toString().charAt(0))
                        && atLeast3Matcher.reset().find()
                        && !textBlock.toString().contains(":") //
-                       || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && atLeast3Matcher.reset().find() && !textBlock.toString().contains(":") //
+                       || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT))
+                          && atLeast3Matcher.reset().find()
+                          && !textBlock.toString().contains(":")
+                          && !textBlock.toString().startsWith("(")//
                        || textBlock.toString().startsWith("APPENDIX") //
                        || textBlock.toString().startsWith("FIGURE") //
                        || textBlock.toString().startsWith("Continued TABLE") //
                        || textBlock.toString().startsWith("TABLE"))
                    && !textBlock.toString().endsWith(":")
                    && atLeast3Matcher.reset().find()) {
-            PageBlockType headlineType = PageBlockType.getHeadlineType(1);
+
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
             document.setHeadlines(true);
-
         } else if (headlineWithIdentifierMatcher.reset().find()
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9
                    && atLeast3Matcher.reset().find()
                    && !headlineWithSlashesMatcher.reset().matches()) {
-            PageBlockType headlineType = PageBlockType.getHeadlineType(2);
+
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
             document.setHeadlines(true);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
@@ -143,9 +147,9 @@ public class DocuMineClassificationService {
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
             textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
         } else {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/HeadlineClassificationService.java

@@ -2,7 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 
 import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.getHeadlineNumber;
 
+import java.util.List;
+
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import lombok.Getter;
@@ -16,6 +19,7 @@ public class HeadlineClassificationService {
     PageBlockType originalClassifiedBlockType;
     TextPageBlock lastHeadlineFromOutline;
 
+
     public void setLastHeadlineFromOutline(TextPageBlock lastHeadlineFromOutline) {
 
         this.lastHeadlineFromOutline = lastHeadlineFromOutline;
@@ -25,28 +29,57 @@ public class HeadlineClassificationService {
 
     public void classifyHeadline(TextPageBlock textBlock, PageBlockType initialHeadlineType) {
 
-        TextPageBlock lastHeadline = getLastHeadline();
-        TextPageBlock lastHeadlineFromOutline = getLastHeadlineFromOutline();
-        PageBlockType originalClassifiedBlockType = getOriginalClassifiedBlockType();
         PageBlockType finalHeadlineType = initialHeadlineType;
 
         if (lastHeadline != null) {
 
-            if (lastHeadline.equals(lastHeadlineFromOutline)) {
-
-                finalHeadlineType = PageBlockType.getHeadlineType(getHeadlineNumber(lastHeadline.getClassification()) + 1);
-
-            } else if (originalClassifiedBlockType != null && lastHeadline.getClassification() != originalClassifiedBlockType) {
-
-                PageBlockType lastHeadlineType = lastHeadline.getClassification();
-                int difference = getHeadlineNumber(originalClassifiedBlockType) - getHeadlineNumber(lastHeadlineType);
-                finalHeadlineType = PageBlockType.getHeadlineType(getHeadlineNumber(initialHeadlineType) - difference);
-            }
+            finalHeadlineType = decideOnClassification(textBlock, initialHeadlineType);
         }
 
-        setOriginalClassifiedBlockType(initialHeadlineType);
+        lastHeadline = textBlock;
+        originalClassifiedBlockType = initialHeadlineType;
         textBlock.setClassification(finalHeadlineType);
-        setLastHeadline(textBlock);
+    }
+
+
+    private PageBlockType decideOnClassification(TextPageBlock textBlock, PageBlockType initialHeadlineType) {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText(textBlock.getText());
+        TextPageBlock lastHeadlineFromOutline = getLastHeadlineFromOutline();
+        PageBlockType originalClassifiedBlockType = getOriginalClassifiedBlockType();
+
+        if (!identifier.isEmpty()) {
+            return PageBlockType.getHeadlineType(identifier.level());
+        }
+
+        if (lastHeadline.equals(lastHeadlineFromOutline) && lastHeadline.getMostPopularWordFontSize() >= textBlock.getMostPopularWordFontSize()) {
+
+            return PageBlockType.getHeadlineType(getHeadlineNumber(lastHeadline.getClassification()) + 1);
+
+        } else if (originalClassifiedBlockType != null && lastHeadline.getClassification() != originalClassifiedBlockType) {
+
+            return adjustInitialLevelToLastHeadlineLevel(initialHeadlineType);
+        }
+        return initialHeadlineType;
+    }
+
+
+    private PageBlockType adjustInitialLevelToLastHeadlineLevel(PageBlockType initialHeadlineType) {
+
+        int difference = getHeadlineNumber(originalClassifiedBlockType) - getHeadlineNumber(lastHeadline.getClassification());
+        return PageBlockType.getHeadlineType(Math.max(1, getHeadlineNumber(initialHeadlineType) - difference));
+    }
+
+
+    public static PageBlockType headlineClassByFontSize(TextPageBlock textBlock, List<Double> fontSizeGroups) {
+
+        PageBlockType headlineType = PageBlockType.H1;
+        for (int i = 1; i <= fontSizeGroups.size(); i++) {
+            if (textBlock.getMostPopularWordFontSize() == fontSizeGroups.get(i - 1)) {
+                headlineType = PageBlockType.getHeadlineType(i);
+            }
+        }
+        return headlineType;
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/RedactManagerClassificationService.java

@@ -22,10 +22,9 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactManagerClassificationService {
 
-
     public void classifyDocument(ClassificationDocument document) {
 
-        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
+        List<Double> headlineFontSizes = document.getFontSizeCounter().getValuesInReverseOrder();
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
@@ -37,7 +36,10 @@ public class RedactManagerClassificationService {
     }
 
 
-    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService,
+                              ClassificationPage page,
+                              ClassificationDocument document,
+                              List<Double> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
@@ -47,7 +49,11 @@ public class RedactManagerClassificationService {
     }
 
 
-    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
+                               TextPageBlock textBlock,
+                               ClassificationPage page,
+                               ClassificationDocument document,
+                               List<Double> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
@@ -56,7 +62,7 @@ public class RedactManagerClassificationService {
             return;
         }
         if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
             return;
         }
 
@@ -71,15 +77,18 @@ public class RedactManagerClassificationService {
             || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
                                                                                                    || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
                 .getMostPopular())) {
+
             textBlock.setClassification(PageBlockType.HEADER);
 
         } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
                    || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null
                                                                                                            || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
                 .getMostPopular())) {
+
             textBlock.setClassification(PageBlockType.FOOTER);
         } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5
                                                  && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
+
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                 textBlock.setClassification(PageBlockType.TITLE);
             }
@@ -88,48 +97,45 @@ public class RedactManagerClassificationService {
                    && (textBlock.getMostPopularWordStyle().equals("bold")
                        || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                           && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getSequences()
-                              .get(0).getTextPositions()
-                              .get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                    PageBlockType headlineType = PageBlockType.getHeadlineType(i);
-                    headlineClassificationService.classifyHeadline(textBlock, headlineType);
-                    document.setHeadlines(true);
-                }
-            }
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
+            headlineClassificationService.classifyHeadline(textBlock, headlineType);
+            document.setHeadlines(true);
         } else if (!textBlock.getText().startsWith("Figure ")
                    && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getSequences()
-                              .get(0).getTextPositions()
-                              .get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-            PageBlockType headlineType = PageBlockType.getHeadlineType(headlineFontSizes.size() + 1);
+                   && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+
+            PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
             document.setHeadlines(true);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+
             textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular())
                    && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular())
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+
             textBlock.setClassification(PageBlockType.PARAGRAPH);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)
                    && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()
                    && textBlock.getMostPopularWordStyle().equals("italic")
                    && !document.getFontStyleCounter().getMostPopular().equals("italic")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+
             textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
+
             textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
         } else {
-            textBlock.setClassification(PageBlockType.OTHER);
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java

@@ -6,6 +6,7 @@ import static java.util.stream.Collectors.toList;
 
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -15,6 +16,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
@@ -32,7 +34,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.He
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableOfContentItem;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
@@ -61,22 +65,38 @@ public class DocumentGraphFactory {
 
         document.getPages()
                 .forEach(context::buildAndAddPageWithCounter);
-        addSections(layoutParsingType, document, context, documentGraph);
+        addSectionsForToC(layoutParsingType, document, context, documentGraph);
         addHeaderAndFooterToEachPage(document, context);
 
         documentGraph.setNumberOfPages(context.pages.size());
         documentGraph.setPages(context.pages.keySet());
         documentGraph.setDocumentTree(context.documentTree);
         documentGraph.setTextBlock(documentGraph.getTextBlock());
+        addTextBlocksToPages(documentGraph);
+
         return documentGraph;
     }
 
 
-    private void addSections(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument, Context context, Document document) {
+    private void addTextBlocksToPages(Document documentGraph) {
+
+        documentGraph.streamAllSubNodes()
+                .filter(SemanticNode::isLeaf)
+                .filter(node -> !node.getType().equals(NodeType.HEADER))
+                .filter(node -> !node.getType().equals(NodeType.FOOTER))
+                .filter(node -> !node.getType().equals(NodeType.IMAGE))
+                .map(SemanticNode::getTextBlock)
+                .map(TextBlock::getAtomicTextBlocks)
+                .flatMap(Collection::stream)
+                .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
+    }
+
+
+    private void addSectionsForToC(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument, Context context, Document document) {
 
         for (TableOfContentItem tocItem : classificationDocument.getTableOfContents()) {
-            var parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
-            Optional<AbstractSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
+            GenericSemanticNode parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
+            Optional<GenericSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
                                                                                    parent,
                                                                                    tocItem.getChildren().isEmpty(),
                                                                                    tocItem.getNonEmptySectionBlocks(),
@@ -105,19 +125,17 @@ public class DocumentGraphFactory {
             node = Paragraph.builder().documentTree(context.getDocumentTree()).build();
         }
 
-        page.getMainBody().add(node);
-
         List<TextPageBlock> textBlocks = new ArrayList<>();
         textBlocks.add(originalTextBlock);
         textBlocks.addAll(textBlocksToMerge);
 
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSort(textBlocks), node, context, page);
 
         if (node instanceof DuplicatedParagraph duplicatedParagraph) {
             AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock(textBlocks.stream()
-                                                                                                      .flatMap(tb -> tb.getSequences()
-                                                                                                              .stream())
-                                                                                                      .collect(Collectors.toList()), node, context, page);
+                                                                                                       .flatMap(tb -> tb.getSequences()
+                                                                                                               .stream())
+                                                                                                       .collect(Collectors.toList()), node, context, page);
             duplicatedParagraph.setUnsortedLeafTextBlock(unsortedTextBlock);
         }
 
@@ -141,7 +159,7 @@ public class DocumentGraphFactory {
 
         Rectangle2D position = image.getPosition();
         Page page = context.getPage(image.getPage());
-        Image imageNode = Image.builder()
+        return Image.builder()
                 .id(IdBuilder.buildId(Set.of(page), List.of(position)))
                 .imageType(image.getImageType())
                 .position(position)
@@ -150,8 +168,6 @@ public class DocumentGraphFactory {
                 .representationHash(image.getRepresentation())
                 .documentTree(context.getDocumentTree())
                 .build();
-        page.getMainBody().add(imageNode);
-        return imageNode;
     }
 
 
@@ -191,7 +207,7 @@ public class DocumentGraphFactory {
 
         Page page = context.getPage(textBlocks.get(0).getPage());
         Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeTextPositionSequence(textBlocks), footer, context, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.merge(textBlocks), footer, context, page);
         List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer);
         footer.setTreeId(tocId);
         footer.setLeafTextBlock(textBlock);
@@ -203,7 +219,7 @@ public class DocumentGraphFactory {
 
         Page page = context.getPage(textBlocks.get(0).getPage());
         Header header = Header.builder().documentTree(context.getDocumentTree()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeTextPositionSequence(textBlocks), header, 0, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.merge(textBlocks), header, 0, page);
         List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header);
         header.setTreeId(tocId);
         header.setLeafTextBlock(textBlock);
@@ -244,7 +260,6 @@ public class DocumentGraphFactory {
         DocumentTree documentTree;
         Map<Page, Integer> pages;
         List<AbstractSemanticNode> sections;
-        List<ClassifiedImage> images;
         TextBlockFactory textBlockFactory;
 
 
@@ -253,7 +268,6 @@ public class DocumentGraphFactory {
             documentTree = new DocumentTree(document);
             pages = new HashMap<>();
             sections = new LinkedList<>();
-            images = new LinkedList<>();
             textBlockFactory = new TextBlockFactory();
         }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SearchTextWithTextPositionFactory.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.factory;
 
-import java.awt.geom.AffineTransform;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition.HEIGHT_PADDING;
+
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.Collections;
@@ -11,7 +12,6 @@ import java.util.Objects;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.experimental.UtilityClass;
@@ -19,14 +19,13 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class SearchTextWithTextPositionFactory {
 
-    public final int HEIGHT_PADDING = 2;
     // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away.
     // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height.
     // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate
     // This is why, we need to initialize this to < -2, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there.
     // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3.
     public final int MAX_HYPHEN_LINEBREAK_DISTANCE = 3;
-    public static final double LINEBREAK_DELTA_TOLERANCE = 1.05;
+    public static final double LINEBREAK_DELTA_TOLERANCE = 1.5;
 
 
     public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<TextPositionSequence> sequences) {
@@ -38,15 +37,13 @@ public class SearchTextWithTextPositionFactory {
 
         Context context = new Context();
 
-        RedTextPosition currentTextPosition = sequences.get(0).getTextPositions()
-                .get(0);
+        RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
         RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").bBoxDirAdj(currentTextPosition.getBBoxDirAdj()).build();
 
         for (TextPositionSequence word : sequences) {
             for (int i = 0; i < word.getTextPositions().size(); ++i) {
 
-                currentTextPosition = word.getTextPositions()
-                        .get(i);
+                currentTextPosition = word.getTextPositions().get(i);
                 if (isLineBreak(currentTextPosition, previousTextPosition)) {
                     removeHyphenLinebreaks(context);
                     context.lineBreaksStringIdx.add(context.stringIdx);
@@ -68,11 +65,10 @@ public class SearchTextWithTextPositionFactory {
             ++context.stringIdx;
         }
 
-
         List<Rectangle2D> positions = sequences.stream()
                 .map(TextPositionSequence::getTextPositions)
                 .flatMap(Collection::stream)
-                .map(RedTextPosition::getBBoxInitialUserSpace)
+                .map(RedTextPosition::getBBoxPdf)
                 .toList();
 
         assert context.stringBuilder.length() == context.stringIdxToPositionIdx.size();
@@ -161,8 +157,8 @@ public class SearchTextWithTextPositionFactory {
             return false;
         }
 
-        double deltaY = Math.abs(currentPosition.getYDirAdj() - previousPosition.getYDirAdj()) * LINEBREAK_DELTA_TOLERANCE;
-        return deltaY >= currentPosition.getHeightDir() || deltaY >= previousPosition.getHeightDir();
+        double deltaY = (Math.abs(currentPosition.getYDirAdj() - previousPosition.getYDirAdj()) * LINEBREAK_DELTA_TOLERANCE) + (2 * HEIGHT_PADDING);
+        return deltaY >= currentPosition.getHeightDirAdj() || deltaY >= previousPosition.getHeightDirAdj();
     }
 
 
@@ -188,32 +184,6 @@ public class SearchTextWithTextPositionFactory {
     }
 
 
-    public Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
-
-        float textHeight = sequence.getTextHeight() + HEIGHT_PADDING;
-        Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(),
-                textPosition.getYDirAdj() - textHeight,
-                textPosition.getWidthDirAdj(),
-                textHeight + HEIGHT_PADDING);
-
-        AffineTransform transform = new AffineTransform();
-
-        if (sequence.getDir() == TextDirection.ZERO || sequence.getDir() == TextDirection.HALF_CIRCLE) {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageWidth() / 2f, sequence.getPageHeight() / 2f);
-            transform.translate(0f, sequence.getPageHeight());
-        } else if (sequence.getDir() == TextDirection.QUARTER_CIRCLE) {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageWidth() / 2f, sequence.getPageWidth() / 2f);
-            transform.translate(0f, sequence.getPageWidth());
-        } else {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageHeight() / 2f, sequence.getPageHeight() / 2f);
-            transform.translate(0f, sequence.getPageWidth());
-        }
-        transform.scale(1., -1.);
-
-        return transform.createTransformedShape(rectangle2D).getBounds2D();
-    }
-
-
     private class Context {
 
         List<Integer> stringIdxToPositionIdx = new LinkedList<>();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SectionNodeFactory.java

@@ -2,13 +2,11 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;
 
 import static java.lang.String.format;
 import static java.util.Collections.emptyList;
-import static java.util.stream.Collectors.groupingBy;
 
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
@@ -17,7 +15,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
@@ -30,13 +27,13 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class SectionNodeFactory {
 
-    public Optional<AbstractSemanticNode> addSection(LayoutParsingType layoutParsingType,
-                                                     GenericSemanticNode parentNode,
-                                                     boolean isLeaf,
-                                                     List<AbstractPageBlock> pageBlocks,
-                                                     List<ClassifiedImage> images,
-                                                     DocumentGraphFactory.Context context,
-                                                     Document document) {
+    public Optional<GenericSemanticNode> addSection(LayoutParsingType layoutParsingType,
+                                                    GenericSemanticNode parentNode,
+                                                    boolean isLeaf,
+                                                    List<AbstractPageBlock> pageBlocks,
+                                                    List<ClassifiedImage> images,
+                                                    DocumentGraphFactory.Context context,
+                                                    Document document) {
 
         // This is for the case where we have images on a page without any text/footer/header.
         // The pageBlocks list is empty, but we still need to add those images to the document.
@@ -51,24 +48,19 @@ public class SectionNodeFactory {
             return Optional.empty();
         }
 
-        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream()
-                .collect(groupingBy(AbstractPageBlock::getPage));
-
         AbstractSemanticNode section;
-        boolean containsTablesAndTextBlocks = containsTablesAndTextBlocks(pageBlocks);
-        if (isLeaf && !containsTablesAndTextBlocks) {
+        if (isLeaf) {
             section = Section.builder().documentTree(context.getDocumentTree()).build();
         } else {
             section = SuperSection.builder().documentTree(context.getDocumentTree()).build();
         }
 
         context.getSections().add(section);
-        blocksPerPage.keySet()
-                .forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));
 
         section.setTreeId(getTreeId(parentNode, context, section));
 
         addFirstHeadlineDirectlyToSection(layoutParsingType, pageBlocks, context, section, document);
+        boolean containsTablesAndTextBlocks = containsTablesAndTextBlocks(pageBlocks);
         if (containsTablesAndTextBlocks) {
             splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(layoutParsingType,
                                                                                                   section,
@@ -158,7 +150,8 @@ public class SectionNodeFactory {
     private boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {
 
         return pageBlocks.stream()
-                       .anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
+                       .anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) //
+               && pageBlocks.stream()
                        .anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
     }
 
@@ -234,18 +227,11 @@ public class SectionNodeFactory {
                 .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
                 .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
                 .filter(abstractTextContainer -> abstractTextContainer instanceof TextPageBlock)
-                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
+                .filter(abstractTextContainer -> abstractTextContainer.intersectsYPdf(atc))
                 .map(abstractTextContainer -> (TextPageBlock) abstractTextContainer)
                 .filter(abstractTextContainer -> abstractTextContainer.getDir() == atc.getDir())
                 .filter(abstractTextContainer -> !abstractTextContainer.isToDuplicate())
                 .toList();
     }
 
-
-    private void addSectionNodeToPageNode(DocumentGraphFactory.Context context, AbstractSemanticNode section, Integer pageNumber) {
-
-        Page page = context.getPage(pageNumber);
-        page.getMainBody().add(section);
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/TableNodeFactory.java

@@ -4,7 +4,6 @@ import static java.util.Collections.emptyList;
 
 import java.util.Collection;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@@ -12,7 +11,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
@@ -36,10 +34,7 @@ public class TableNodeFactory {
                          Document document) {
 
         setPageNumberInCells(tablesToMerge);
-        Set<Page> pages = tablesToMerge.stream()
-                .map(AbstractPageBlock::getPage)
-                .map(context::getPage)
-                .collect(Collectors.toSet());
+
         List<List<Cell>> mergedRows = tablesToMerge.stream()
                 .map(TablePageBlock::getRows)
                 .flatMap(Collection::stream)
@@ -51,8 +46,6 @@ public class TableNodeFactory {
                 .numberOfRows(mergedRows.size())
                 .build();
 
-        pages.forEach(page -> addTableToPage(page, parentNode, table));
-
         List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table);
         table.setTreeId(treeId);
         addTableCells(layoutParsingType, mergedRows, table, context, document);
@@ -82,17 +75,6 @@ public class TableNodeFactory {
     }
 
 
-    @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong
-    private void addTableToPage(Page page, SemanticNode parentNode, Table table) {
-
-        if (!page.getMainBody().contains(parentNode)) {
-            parentNode.getPages().add(page);
-        }
-
-        page.getMainBody().add(table);
-    }
-
-
     private void ifTableHasNoHeadersSetFirstRowAsHeaders(Table table) {
 
         if (table.streamHeaders()
@@ -107,14 +89,7 @@ public class TableNodeFactory {
 
         for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
             for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(layoutParsingType,
-                             rows.get(rowIndex)
-                                     .get(colIndex),
-                             rowIndex,
-                             colIndex,
-                             table,
-                             context,
-                             document);
+                addTableCell(layoutParsingType, rows.get(rowIndex).get(colIndex), rowIndex, colIndex, table, context, document);
             }
         }
     }
@@ -131,14 +106,7 @@ public class TableNodeFactory {
 
         Page page = context.getPage(cell.getPageNumber());
 
-        TableCell tableCell = TableCell.builder()
-                .documentTree(context.getDocumentTree())
-                .row(rowIndex)
-                .col(colIndex)
-                .header(cell.isHeaderCell())
-                .bBox(cell.getBBoxInitialUserSpace())
-                .build();
-        page.getMainBody().add(tableCell);
+        TableCell tableCell = TableCell.builder().documentTree(context.getDocumentTree()).row(rowIndex).col(colIndex).header(cell.isHeaderCell()).bBox(cell.getBBoxPdf()).build();
 
         List<Integer> treeId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell);
         tableCell.setTreeId(treeId);
@@ -147,9 +115,7 @@ public class TableNodeFactory {
         if (cell.getTextBlocks().isEmpty()) {
             tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page));
         } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.getTextBlockFactory()
-                    .buildAtomicTextBlock(cell.getTextBlocks()
-                                                  .get(0).getSequences(), tableCell, context, page);
+            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
         } else if (firstTextBlockIsHeadline(cell)) {
             SectionNodeFactory.addSection(layoutParsingType,
@@ -163,7 +129,7 @@ public class TableNodeFactory {
                                           context,
                                           document);
         } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
-            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
+            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSort(cell.getTextBlocks());
             textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
         } else {
@@ -181,8 +147,7 @@ public class TableNodeFactory {
 
     private boolean firstTextBlockIsHeadline(Cell cell) {
 
-        return cell.getTextBlocks()
-                .get(0).isHeadline();
+        return cell.getTextBlocks().get(0).isHeadline();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/TextBlockFactory.java

@@ -32,35 +32,34 @@ public class TextBlockFactory {
         long idx = textBlockIdx;
         textBlockIdx++;
         String orientation;
-        int textDirection;
+        int textRotation;
         if (sequences.isEmpty()) {
             orientation = null;
-            textDirection = 0;
+            textRotation = 0;
         } else {
             orientation = sequences.get(0).getDir().toString();
-            textDirection = sequences.get(0).getRotation();
+            textRotation = sequences.get(0).getDir().getRotation();
         }
-        return AtomicTextBlock.fromSearchTextWithTextPosition(searchTextWithTextPositionDto.getSearchText(),
-                searchTextWithTextPositionDto.getLineBreaks(),
-                searchTextWithTextPositionDto.getBoldTextBoundaries(),
-                searchTextWithTextPositionDto.getItalicTextBoundaries(),
-                searchTextWithTextPositionDto.getPositions(),
-                searchTextWithTextPositionDto.getStringIdxToPositionIdx(),
-                idx,
-                parent,
-                numberOnPage,
-                page,
-                offset,
-                orientation,
-                textDirection);
+        var atb = AtomicTextBlock.fromSearchTextWithTextPosition(searchTextWithTextPositionDto.getSearchText(),
+                                                                 searchTextWithTextPositionDto.getLineBreaks(),
+                                                                 searchTextWithTextPositionDto.getBoldTextBoundaries(),
+                                                                 searchTextWithTextPositionDto.getItalicTextBoundaries(),
+                                                                 searchTextWithTextPositionDto.getPositions(),
+                                                                 searchTextWithTextPositionDto.getStringIdxToPositionIdx(),
+                                                                 idx,
+                                                                 parent,
+                                                                 numberOnPage,
+                                                                 page,
+                                                                 offset,
+                                                                 orientation,
+                                                                 textRotation);
+        return atb;
     }
 
 
     public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
 
-        long idx = textBlockIdx;
-        textBlockIdx++;
-        return AtomicTextBlock.empty(idx, stringOffset, page, context.getAndIncrementTextBlockNumberOnPage(page), parent);
+        return emptyTextBlock(parent, context.getAndIncrementTextBlockNumberOnPage(page), page);
     }
 
 
@@ -68,7 +67,8 @@ public class TextBlockFactory {
 
         long idx = textBlockIdx;
         textBlockIdx++;
-        return AtomicTextBlock.empty(idx, stringOffset, page, numberOnPage, parent);
+        var atb = AtomicTextBlock.empty(idx, stringOffset, page, numberOnPage, parent);
+        return atb;
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/GraphicExtractorService.java

@@ -8,12 +8,11 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -67,10 +66,7 @@ public class GraphicExtractorService {
     private List<Box> getCharacterBBoxes(List<TextPositionSequence> textPositionSequences) {
 
         return textPositionSequences.stream()
-                .map(pos -> pos.getTextPositions()
-                        .stream()
-                        .map(tp -> SearchTextWithTextPositionFactory.mapRedTextPositionToInitialUserSpace(tp, pos))
-                        .collect(RectangleTransformations.collectBBox()))
+                .map(BoundingBox::getBBoxPdf)
                 .map(Box::new)
                 .collect(Collectors.toList());
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentGraphMapper.java

@@ -41,7 +41,9 @@ public class DocumentGraphMapper {
         DocumentTree documentTree = new DocumentTree(document);
         Context context = new Context(documentData, documentTree);
 
-        context.pages.addAll(Arrays.stream(documentData.getDocumentPages()).map(DocumentGraphMapper::buildPage).toList());
+        context.pages.addAll(Arrays.stream(documentData.getDocumentPages())
+                                     .map(DocumentGraphMapper::buildPage)
+                                     .toList());
 
         context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
 
@@ -59,7 +61,9 @@ public class DocumentGraphMapper {
         List<DocumentTree.Entry> newEntries = new LinkedList<>();
         for (DocumentStructure.EntryData entryData : entries) {
 
-            List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();
+            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
+                    .map(pageNumber -> getPage(pageNumber, context))
+                    .toList();
 
             SemanticNode node = switch (entryData.getType()) {
                 case SECTION -> buildSection(context);
@@ -77,16 +81,18 @@ public class DocumentGraphMapper {
             if (entryData.getAtomicBlockIds().length > 0) {
                 TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
                 node.setLeafTextBlock(textBlock);
+                switch (entryData.getType()) {
+                    case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
+                    case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
+                    case IMAGE -> pages.forEach(page -> page.getImages().add((Image) node));
+                    default -> textBlock.getAtomicTextBlocks()
+                            .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
+                }
             }
-            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
+            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
+                    .toList();
             node.setTreeId(treeId);
 
-            switch (entryData.getType()) {
-                case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
-                case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
-                default -> pages.forEach(page -> page.getMainBody().add(node));
-            }
-
             newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
         }
         return newEntries;
@@ -142,6 +148,7 @@ public class DocumentGraphMapper {
         return Section.builder().documentTree(context.documentTree).build();
     }
 
+
     private SuperSection buildSuperSection(Context context) {
 
         return SuperSection.builder().documentTree(context.documentTree).build();
@@ -166,22 +173,24 @@ public class DocumentGraphMapper {
 
     private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
 
-        return Arrays.stream(atomicTextBlockIds).map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)).collect(new TextBlockCollector());
+        return Arrays.stream(atomicTextBlockIds)
+                .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
+                .collect(new TextBlockCollector());
     }
 
 
     private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
 
         return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                parent,
-                getPage(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                                                       context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
+                                                       parent,
+                                                       getPage(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
     }
 
 
     private Page buildPage(DocumentPage p) {
 
-        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
+        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).textBlocksOnPage(new LinkedList<>()).build();
     }
 
 
@@ -206,8 +215,10 @@ public class DocumentGraphMapper {
 
             this.documentTree = documentTree;
             this.pages = new LinkedList<>();
-            this.documentTextDataBlockData = Arrays.stream(documentData.getDocumentTextData()).toList();
-            this.atomicPositionBlockData = Arrays.stream(documentData.getDocumentPositions()).toList();
+            this.documentTextDataBlockData = Arrays.stream(documentData.getDocumentTextData())
+                    .toList();
+            this.atomicPositionBlockData = Arrays.stream(documentData.getDocumentPositions())
+                    .toList();
 
         }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/MarkdownMapper.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.markdown;
+package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -32,6 +32,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.He
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;
 
 public class MarkdownMapper extends AbstractNodeVisitor {
 
@@ -297,12 +298,6 @@ public class MarkdownMapper extends AbstractNodeVisitor {
     }
 
 
-    enum FontStyle {
-        REGULAR,
-        BOLD,
-        ITALIC,
-        BOLD_ITALIC;
-    }
 
     record FontStyleChange(boolean enter, FontStyle style) {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/OutlineMapper.java

@@ -0,0 +1,84 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
+
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Optional;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+import com.knecon.fforesight.service.viewerdoc.model.Outline;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class OutlineMapper {
+
+    public Outline createOutline(Document document) {
+
+        Outline outline = new Outline();
+        addChildren(document, null, outline);
+        return outline;
+    }
+
+
+    public void addChildren(SemanticNode parentNode, Outline.Entry parentEntry, Outline outline) {
+
+        parentNode.streamChildren()
+                .filter(child -> child instanceof Section || child instanceof SuperSection)
+                .forEach(child -> {
+                    Optional<Headline> headline = findHeadline(child);
+                    if (headline.isPresent()) {
+                        Outline.Entry entry = buildEntry(child.getHeadline());
+                        if (parentEntry != null) {
+                            parentEntry.children().add(entry);
+                        } else {
+                            outline.getEntries().add(entry);
+                        }
+                        addChildren(child, entry, outline);
+                    } else {
+                        addChildren(child, parentEntry, outline);
+                    }
+                });
+    }
+
+
+    private static Optional<Headline> findHeadline(SemanticNode child) {
+
+        return child.streamChildren()
+                .filter(node -> node instanceof Headline)
+                .map(node -> (Headline) node)
+                .findFirst();
+    }
+
+
+    @SneakyThrows
+    private Outline.Entry buildEntry(Headline headline) {
+
+        Map<Page, Rectangle2D> bbox = headline.getBBox();
+        Rectangle2D r = bbox.get(headline.getFirstPage());
+        Point2D.Double position = new Point2D.Double(r.getMinX(), r.getMaxY());
+        PageInformation pageInformation = PageInformation.fromPage(headline.getFirstPage());
+
+        AffineTransform pdfToPage = CoordinateTransforms.calculateInitialUserSpaceCoordsToPageCoords(pageInformation);
+        pdfToPage.transform(position, position);
+
+        AffineTransform mirror = new AffineTransform(1, 0, 0, -1, 0, pageInformation.heightRot());
+        mirror.transform(position, position);
+
+        AffineTransform.getTranslateInstance(0, 5).transform(position, position);
+
+        Outline.JumpAction action = new Outline.JumpAction(headline.getFirstPage().getNumber(), position);
+        return new Outline.Entry(headline.getTextBlock().getSearchText(), action, new LinkedList<>());
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/PDFTextStripper.java

@@ -25,10 +25,22 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.regex.Pattern;
 
-import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.pdfbox.cos.COSDictionary;
@@ -46,6 +58,8 @@ import org.apache.pdfbox.text.TextPositionComparator;
 
 import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;
 
+import lombok.Getter;
+
 /**
  * This is just a copy except i only adjusted lines 594-607 cause this is a bug in Pdfbox.
  * see S416.pdf
@@ -194,40 +208,33 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
     }
 
 
+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {
 
-    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
-    {
         PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
-        if (this.currentMarkedContents.isEmpty())
-        {
+        if (this.currentMarkedContents.isEmpty()) {
             this.markedContents.add(markedContent);
-        }
-        else
-        {
-            PDMarkedContent currentMarkedContent =
-                    this.currentMarkedContents.peek();
-            if (currentMarkedContent != null)
-            {
+        } else {
+            PDMarkedContent currentMarkedContent = this.currentMarkedContents.peek();
+            if (currentMarkedContent != null) {
                 currentMarkedContent.addMarkedContent(markedContent);
             }
         }
         this.currentMarkedContents.push(markedContent);
     }
 
+
     @Override
-    public void endMarkedContentSequence()
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void endMarkedContentSequence() {
+
+        if (!this.currentMarkedContents.isEmpty()) {
             this.currentMarkedContents.pop();
         }
     }
 
 
-    public void xobject(PDXObject xobject)
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void xobject(PDXObject xobject) {
+
+        if (!this.currentMarkedContents.isEmpty()) {
             this.currentMarkedContents.peek().addXObject(xobject);
         }
     }
@@ -313,7 +320,11 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             endBookmarkPageNumber = -1;
         }
 
-        if (startBookmarkPageNumber == -1 && startBookmark != null && endBookmarkPageNumber == -1 && endBookmark != null && startBookmark.getCOSObject() == endBookmark.getCOSObject()) {
+        if (startBookmarkPageNumber == -1
+            && startBookmark != null
+            && endBookmarkPageNumber == -1
+            && endBookmark != null
+            && startBookmark.getCOSObject() == endBookmark.getCOSObject()) {
             // this is a special case where both the start and end bookmark
             // are the same but point to nothing. In this case
             // we will not extract any text.
@@ -360,7 +371,9 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
     @Override
     public void processPage(PDPage page) throws IOException {
 
-        if (currentPageNo >= startPage && currentPageNo <= endPage && (startBookmarkPageNumber == -1 || currentPageNo >= startBookmarkPageNumber) && (endBookmarkPageNumber == -1 || currentPageNo <= endBookmarkPageNumber)) {
+        if (currentPageNo >= startPage && currentPageNo <= endPage && (startBookmarkPageNumber == -1 || currentPageNo >= startBookmarkPageNumber) && (endBookmarkPageNumber == -1
+                                                                                                                                                      || currentPageNo
+                                                                                                                                                         <= endBookmarkPageNumber)) {
             startPage(page);
 
             int numberOfArticleSections = 1;
@@ -635,7 +648,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                         var normalized = normalize(line);
 //                        normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent()
 
-
                         lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine);
                         writeLine(normalized, current.isParagraphStart);
                         line.clear();
@@ -647,8 +659,8 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                     }
                     // test if our TextPosition starts after a new word would be expected to start
                     if (expectedStartOfNextWordX != EXPECTED_START_OF_NEXT_WORD_X_RESET_VALUE && expectedStartOfNextWordX < positionX
-                            // only bother adding a word separator if the last character was not a word separator
-                            && (wordSeparator.isEmpty() || //
+                        // only bother adding a word separator if the last character was not a word separator
+                        && (wordSeparator.isEmpty() || //
                             (lastPosition.getTextPosition().getUnicode() != null && !lastPosition.getTextPosition().getUnicode().endsWith(wordSeparator)))) {
                         line.add(LineItem.getWordSeparator());
                     }
@@ -914,8 +926,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                     textList.add(text);
                 }
             }
-            if (!this.currentMarkedContents.isEmpty())
-            {
+            if (!this.currentMarkedContents.isEmpty()) {
                 this.currentMarkedContents.peek().addText(text);
             }
         }
@@ -1711,7 +1722,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
         int numberOfStrings = line.size();
         for (int i = 0; i < numberOfStrings; i++) {
             WordWithTextPositions word = line.get(i);
-            word.getTextPositions().sort(Comparator.comparing(TextPosition::getXDirAdj));
             writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
             if (i < numberOfStrings - 1) {
                 writeWordSeparator();
@@ -2102,7 +2112,9 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             return endParagraphWritten;
         }
 
-        public void setEndParagraphWritten(){
+
+        public void setEndParagraphWritten() {
+
             endParagraphWritten = true;
         }
 
@@ -2145,7 +2157,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             this.isHangingIndent = true;
         }
 
-
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java

@@ -10,7 +10,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.He
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.OutlineMapper;
 import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutGrid;
+import com.knecon.fforesight.service.viewerdoc.model.Outline;
 import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;
 
 import io.micrometer.observation.annotation.Observed;
@@ -29,16 +31,15 @@ public class LayoutGridService {
 
     @SneakyThrows
     @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
-    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue, boolean writeVisualLayoutParsingGrid) {
+    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {
 
         LayoutGrid layoutGrid = createLayoutGrid(document);
+        Outline outline = OutlineMapper.createOutline(document);
         layoutGrid.setVisibleByDefault(layerVisibilityDefaultValue);
-//        Visualizations visualLayoutGrid = this.addLayoutGrid(document, layerVisibilityDefaultValue, true);
         if (document.getLayoutDebugLayer().isActive()) {
-            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid, document.getLayoutDebugLayer()));
+            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid, document.getLayoutDebugLayer()), outline);
         } else {
-            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid));
-
+            viewerDocumentService.addLayerGroups(originFile, destinationFile, List.of(layoutGrid), outline);
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/CoordinateTransforms.java

@@ -32,7 +32,6 @@ public class CoordinateTransforms {
     }
 
 
-
     @SneakyThrows
     public AffineTransform calculateInitialUserSpaceCoordsToImageCoords(PageInformation pageInformation, double scalingFactor) {
 
@@ -40,6 +39,19 @@ public class CoordinateTransforms {
     }
 
 
+    public AffineTransform calculatePageCoordsToInitialUserSpaceCoords(PageInformation pageInformation) {
+
+        return calculateImageCoordsToInitialUserSpaceCoords(pageInformation, 1);
+    }
+
+
+    @SneakyThrows
+    public AffineTransform calculateInitialUserSpaceCoordsToPageCoords(PageInformation pageInformation) {
+
+        return calculatePageCoordsToInitialUserSpaceCoords(pageInformation).createInverse();
+    }
+
+
     public double calculateScalingFactor(PageInformation pageInformation, double imageWidth) {
 
         // PDFBox always returns page height and width based on rotation

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/HeaderFooterDetection.java

@@ -96,7 +96,7 @@ public class HeaderFooterDetection {
                         continue;
                     }
 
-                    int distance = calculateHammingDistanceWithPreprocessing(testString, paddedString);
+                    int distance = StringDistances.hamming(testString, paddedString);
                     double normalizedScore = 1 - (double) distance / Math.max(testString.length(), paddedString.length());
                     score += normalizedScore * (j < weights.length ? weights[j] : 1);
                 }
@@ -180,44 +180,4 @@ public class HeaderFooterDetection {
         return headerCandidates;
     }
 
-
-    /**
-     * Calculate the Hamming distance between two strings after preprocessing to make them the same length
-     * and replacing all digits with a special character '@' since they are a common occurrence in headers/footers.
-     *
-     * @param firstCandidate  First string
-     * @param secondCandidate Second string
-     * @return The Hamming distance between the two preprocessed strings.
-     */
-    private int calculateHammingDistanceWithPreprocessing(String firstCandidate, String secondCandidate) {
-
-        int maxLength = Math.max(firstCandidate.length(), secondCandidate.length());
-
-        String cleanFirstCandidate = padString(firstCandidate, maxLength, '\0').replaceAll("\\d", "@");
-        String cleanSecondCandidate = padString(secondCandidate, maxLength, '\0').replaceAll("\\d", "@");
-
-        int distance = 0;
-        for (int i = 0; i < maxLength; i++) {
-            if (cleanFirstCandidate.charAt(i) != cleanSecondCandidate.charAt(i)) {
-                distance++;
-            }
-        }
-        return distance;
-    }
-
-
-    private String padString(String input, int length, char padChar) {
-
-        if (input.length() >= length) {
-            return input;
-        }
-
-        StringBuilder sb = new StringBuilder(input);
-
-        while (sb.length() < length) {
-            sb.append(padChar);
-        }
-        return sb.toString();
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/MarkedContentUtils.java

@@ -48,7 +48,7 @@ public class MarkedContentUtils {
 
         return markedContentByYPosition.values()
                 .stream()
-                .map(textPositions -> new TextPositionSequence(textPositions, 0, true).getBBoxInitialUserSpace())
+                .map(textPositions -> new TextPositionSequence(textPositions, 0, true).getBBoxPdf())
                 .map(t -> new Rectangle2D.Double(t.getX(), t.getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight())))
                 .collect(Collectors.toList());
     }
@@ -90,7 +90,7 @@ public class MarkedContentUtils {
                     .map(content -> (TextPosition) content)
                     .filter(content -> !content.getUnicode().equals(" "))
                     .map(textPositions -> new TextPositionSequence(List.of(textPositions), 0, true))
-                    .map(BoundingBox::getBBoxInitialUserSpace)
+                    .map(BoundingBox::getBBoxPdf)
                     .collect(Collectors.toList());
         }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/PageInformation.java

@@ -5,14 +5,22 @@ import java.awt.geom.Rectangle2D;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
 
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+
 public record PageInformation(Rectangle2D mediabox, int number, int rotationDegrees) {
 
     public static PageInformation fromPDPage(int pageNum, PDPage page) {
 
         PDRectangle mediaBox = page.getMediaBox();
         return new PageInformation(new Rectangle2D.Double(mediaBox.getLowerLeftX(), mediaBox.getLowerLeftY(), mediaBox.getWidth(), mediaBox.getHeight()),
-                pageNum,
-                page.getRotation());
+                                   pageNum,
+                                   page.getRotation());
+    }
+
+
+    public static PageInformation fromPage(Page page) {
+
+        return new PageInformation(new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight()), page.getNumber(), page.getRotation());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/PositionUtils.java

@@ -108,7 +108,7 @@ public final class PositionUtils {
     }
 
 
-    public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextPageBlock textBlock, Float documentMostPopularWordHeight) {
+    public double getHeightDifferenceBetweenChunkWordAndDocumentWord(TextPageBlock textBlock, Double documentMostPopularWordHeight) {
 
         return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
     }
@@ -116,7 +116,7 @@ public final class PositionUtils {
 
     public double getApproxLineCount(TextPageBlock textBlock) {
 
-        return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
+        return textBlock.getBBoxDirAdj().getHeight() / textBlock.getMostPopularWordHeight();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/SpreadsheetFinder.java

@@ -28,7 +28,7 @@ public class SpreadsheetFinder {
         Map<Point2D, Point2D> edgesV = new HashMap<>();
 
         for (Cell cell : cells) {
-            for (Point2D pt : getPoints(cell.getBBoxInitialUserSpace())) {
+            for (Point2D pt : getPoints(cell.getBBoxPdf())) {
                 if (pointSet.contains(pt)) { // shared vertex, remove it
                     pointSet.remove(pt);
                 } else {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/StringDistances.java

@@ -0,0 +1,49 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class StringDistances {
+
+
+    /**
+     * Calculate the Hamming distance between two strings after preprocessing to make them the same length
+     * and replacing all digits with a special character '@' since they are a common occurrence in headers/footers.
+     *
+     * @param s1  First string
+     * @param s2 Second string
+     * @return The Hamming distance between the two preprocessed strings.
+     */
+    public int hamming(String s1, String s2) {
+
+        int maxLength = Math.max(s1.length(), s2.length());
+
+        String cleanFirstCandidate = padString(s1, maxLength, '\0').replaceAll("\\d", "@");
+        String cleanSecondCandidate = padString(s2, maxLength, '\0').replaceAll("\\d", "@");
+
+        int distance = 0;
+        for (int i = 0; i < maxLength; i++) {
+            if (cleanFirstCandidate.charAt(i) != cleanSecondCandidate.charAt(i)) {
+                distance++;
+            }
+        }
+        return distance;
+    }
+
+
+    private String padString(String input, int length, char padChar) {
+
+        if (input.length() >= length) {
+            return input;
+        }
+
+        StringBuilder sb = new StringBuilder(input);
+
+        while (sb.length() < length) {
+            sb.append(padChar);
+        }
+        return sb.toString();
+    }
+
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TableMergingUtility.java

@@ -1,9 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;
 
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.stream.Stream;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -22,29 +23,77 @@ public class TableMergingUtility {
         List<TablePageBlock> consecutiveTables = pageBlocks.stream()
                 .map(abstractPageBlock -> (TablePageBlock) abstractPageBlock)
                 .filter(tablePageBlock -> !tablePageBlock.equals(originalTablePageBlock))
+                .sorted(Comparator.comparingInt(TablePageBlock::getPage).thenComparing(TablePageBlock::getY).thenComparing(TablePageBlock::getX))
                 .toList();
+
         assert consecutiveTables.size() == pageBlocks.size() - 1;
+        var currentTable = originalTablePageBlock;
+        int currentTableIndex = 0;
 
         List<TablePageBlock> consecutiveTablesWithSameColCountAndHeaders = new LinkedList<>();
-        for (TablePageBlock consecutiveTable : consecutiveTables) {
-            if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() && !hasTableHeader(consecutiveTable) && outerBoundaryAlignsX(originalTablePageBlock,
-                    consecutiveTable)) {
+        consecutiveTablesWithSameColCountAndHeaders.add(originalTablePageBlock);
+        for (int i = 0; i < consecutiveTables.size(); i++) {
+            TablePageBlock consecutiveTable = consecutiveTables.get(i);
+
+            if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() //
+                && getHeaders(consecutiveTable).isEmpty() //
+                && outerBoundaryAlignsX(originalTablePageBlock, consecutiveTable) //
+                && consecutiveOrSamePage(currentTable, consecutiveTable) //
+                && !tableBetween(currentTable, consecutiveTable, findTablesBetween(consecutiveTables, currentTableIndex, i))) {
+
+                currentTable = consecutiveTable;
+                currentTableIndex = i;
                 consecutiveTablesWithSameColCountAndHeaders.add(consecutiveTable);
             }
         }
-        return Stream.concat(Stream.of(originalTablePageBlock), consecutiveTablesWithSameColCountAndHeaders.stream()).toList();
+        return consecutiveTablesWithSameColCountAndHeaders;
+    }
+
+
+    private static List<TablePageBlock> findTablesBetween(List<TablePageBlock> consecutiveTables, int currentTableIndex, int i) {
+
+        if (currentTableIndex + 1 == consecutiveTables.size() || currentTableIndex + 1 >= i) {
+            return Collections.emptyList();
+        }
+        return consecutiveTables.subList(currentTableIndex + 1, i);
+    }
+
+
+    private static boolean consecutiveOrSamePage(TablePageBlock currentTable, TablePageBlock consecutiveTable) {
+
+        return currentTable.getPage() == consecutiveTable.getPage() || currentTable.getPage() + 1 == consecutiveTable.getPage();
+    }
+
+
+    private static boolean tableBetween(TablePageBlock currentTable, TablePageBlock consecutiveTable, List<TablePageBlock> tablesBetween) {
+
+        if (tablesBetween.isEmpty()) {
+            return false;
+        }
+        // assumes the tables are on the same page or on consecutive pages, all tables on pages in between are ignored.
+        return tablesBetween.stream()
+                       .filter(tableBetween -> tableBetween.getPage() == currentTable.getPage())
+                       .anyMatch(tableBetween -> tableBetween.isBelow(currentTable)) //
+               || tablesBetween.stream()
+                       .filter(tableBetween -> tableBetween.getPage() == consecutiveTable.getPage())
+                       .anyMatch(tableBetween -> tableBetween.isAbove(consecutiveTable));
     }
 
 
     private static boolean outerBoundaryAlignsX(TablePageBlock originalTablePageBlock, TablePageBlock consecutiveTable) {
 
-        return Math.abs(consecutiveTable.getMinX() - originalTablePageBlock.getMinX()) < TABLE_ALIGNMENT_THRESHOLD && Math.abs(consecutiveTable.getMaxX() - originalTablePageBlock.getMaxX()) < TABLE_ALIGNMENT_THRESHOLD;
+        return Math.abs(consecutiveTable.getMinX() - originalTablePageBlock.getMinX()) < TABLE_ALIGNMENT_THRESHOLD
+               && Math.abs(consecutiveTable.getMaxX() - originalTablePageBlock.getMaxX()) < TABLE_ALIGNMENT_THRESHOLD;
     }
 
 
-    private boolean hasTableHeader(TablePageBlock table) {
+    private List<Cell> getHeaders(TablePageBlock table) {
 
-        return table.getRows().stream().flatMap(Collection::stream).anyMatch(Cell::isHeaderCell);
+        return table.getRows()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(Cell::isHeaderCell)
+                .toList();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextNormalizationUtilities.java

@@ -1,31 +1,39 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;
 
+import java.util.regex.Pattern;
+
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
 public final class TextNormalizationUtilities {
 
-    /**
-     * Revert hyphenation due to line breaks.
-     *
-     * @param text Text to be processed.
-     * @return Text without line-break hyphenation.
-     */
-    public static String removeHyphenLineBreaks(String text) {
+    public static final Pattern hyphenLineBreaks = Pattern.compile("[-~‐‒⁻−﹣゠⁓‑\\u00AD][\\r\\n]+");
+    public static final Pattern linebreaks = Pattern.compile("[\\r\\n]+");
+    public static final Pattern doubleWhitespaces = Pattern.compile("\\s{2,}");
 
-        return text.replaceAll("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R", "$1");
+
+    public String cleanString(String value) {
+
+        String noHyphenLinebreaks = removeHyphenLinebreaks(value);
+        String noLinebreaks = removeLinebreaks(noHyphenLinebreaks);
+        return removeMultipleWhitespaces(noLinebreaks);
     }
 
 
-    public static String removeLineBreaks(String text) {
+    public String removeHyphenLinebreaks(String value) {
 
-        return text.replaceAll("\n", " ");
+        return hyphenLineBreaks.matcher(value).replaceAll("");
     }
 
 
-    public static String removeRepeatingWhitespaces(String text) {
+    private String removeMultipleWhitespaces(String value) {
 
-        return text.replaceAll(" {2}", " ");
+        return doubleWhitespaces.matcher(value).replaceAll(" ");
     }
 
+
+    private String removeLinebreaks(String value) {
+
+        return linebreaks.matcher(value).replaceAll(" ");
+    }
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionOperations.java

@@ -1,30 +1,143 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;
 
+import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
 public class TextPositionOperations {
 
-    private static final TextPositionSequenceComparator comparator = new TextPositionSequenceComparator();
+    public static final double ANGLE_TOLERANCE = Math.PI / 35;
+    public static final AngleFilter ANGLE_FILTER = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
+    public static final double MAX_LINE_HEIGHT_FACTOR = 0.66; // multiplied with max word height
+    public static final double MAX_WORD_DISTANCE_FACTOR = 3.5; // multiplied with max word width
+
+    private static final double Y_THRESHOLD = 5;
+    private static final double X_THRESHOLD = 3;
+    private static final Comparator<TextBoundingBox> COMPARATOR_DIR_ADJ = //
+            Comparator.comparing(TextBoundingBox::getDir)
+                    .thenComparing(TextBoundingBox::getMaxYDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, Y_THRESHOLD))
+                    .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, X_THRESHOLD));
 
 
-    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<TextPageBlock> textBlocks) {
+    public List<TextPositionSequence> mergeAndSort(List<TextPageBlock> textBlocks) {
 
-        var sequence = textBlocks.stream().flatMap(tb -> tb.getSequences().stream()).collect(Collectors.toList());
-
-        // because the TextPositionSequenceComparator is not transitive, but
-        // JDK7+ enforces transitivity on comparators, we need to use
-        // a custom quicksort implementation (which is slower, unfortunately).
-        QuickSort.sort(sequence, comparator);
-        return sequence;
+        var sequences = textBlocks.stream()
+                .flatMap(tb -> tb.getSequences()
+                        .stream())
+                .collect(Collectors.toSet());
+        return sortUsingLineDetection(sequences);
     }
 
-    public static List<TextPositionSequence> mergeTextPositionSequence(List<TextPageBlock> textBlocks) {
 
-       return textBlocks.stream().flatMap(tb -> tb.getSequences().stream()).collect(Collectors.toList());
+    public List<TextPositionSequence> sort(List<TextPositionSequence> sequences) {
+
+        return sortUsingLineDetection(new HashSet<>(sequences));
+    }
+
+
+    private List<TextPositionSequence> sortUsingLineDetection(Set<TextPositionSequence> sequences) {
+
+        return sortLines(groupByLine(sequences));
+
+    }
+
+
+    public List<TextPositionSequence> sortLines(Collection<Set<TextPositionSequence>> lines) {
+
+        return lines.stream()
+                .map(TextPositionOperations::sortByXDirAdj)
+                .filter(line -> !line.isEmpty())
+                .sorted(Comparator.comparing(line -> line.get(0), COMPARATOR_DIR_ADJ))
+                .flatMap(Collection::stream)
+                .toList();
+    }
+
+
+    private List<TextPositionSequence> sortByXDirAdj(Set<TextPositionSequence> line) {
+
+        return line.stream()
+                .sorted(Comparator.comparing(TextPositionSequence::getXDirAdj))
+                .toList();
+    }
+
+
+    public Collection<Set<TextPositionSequence>> groupByLine(Set<TextPositionSequence> sequences) {
+
+        double maxLineDistance = sequences.stream()
+                                         .map(TextPositionSequence::getBBoxDirAdj)
+                                         .mapToDouble(RectangularShape::getHeight).average().orElse(10) * MAX_LINE_HEIGHT_FACTOR;
+        double maxXGap = sequences.stream()
+                                 .map(TextPositionSequence::getBBoxDirAdj)
+                                 .mapToDouble(RectangularShape::getWidth).average().orElse(75) * MAX_WORD_DISTANCE_FACTOR;
+
+        UnionFind<TextPositionSequence> unionFind = new UnionFind<>(sequences);
+
+        for (TextPositionSequence sequence : sequences) {
+            for (TextPositionSequence sequence2 : sequences) {
+
+                if (sequence.equals(sequence2) || unionFind.inSameSet(sequence, sequence2)) {
+                    continue;
+                }
+
+                double angle = computeAngle(sequence.getBBoxDirAdj(), sequence2.getBBoxDirAdj());
+
+                double normalizedVerticalDistance = Math.abs(sequence.getBBoxDirAdj().getCenterY() - sequence2.getBBoxDirAdj().getCenterY()) / maxLineDistance;
+                double normalizedHorizontalDistance = Math.abs(sequence.getBBoxDirAdj().getCenterX() - sequence2.getBBoxDirAdj().getCenterX()) / maxXGap;
+
+                if (sequence.getDir() != sequence2.getDir()
+                    || Math.abs(sequence.getFontSize() - sequence2.getFontSize()) > 0.5 * Math.min(sequence.getFontSize(),
+                                                                                                   sequence2.getFontSize())
+                    || Math.pow(normalizedVerticalDistance, 2) + Math.pow(normalizedHorizontalDistance, 2) > 1
+                    || !ANGLE_FILTER.matches(angle)) {
+                    continue;
+                }
+
+                unionFind.union(sequence, sequence2);
+            }
+        }
+
+        return unionFind.getGroups();
+    }
+
+
+    public double computeAngle(Rectangle2D rect1, Rectangle2D rect2) {
+
+        double rect1CentroidX = rect1.getCenterX();
+        double rect1CentroidY = rect1.getCenterY();
+        double rect2CentroidX = rect2.getCenterX();
+        double rect2CentroidY = rect2.getCenterY();
+
+        double deltaX = rect2CentroidX - rect1CentroidX;
+        double deltaY = rect2CentroidY - rect1CentroidY;
+
+        return FastAtan2.fastAtan2(deltaY, deltaX);
+    }
+
+
+    public List<TextPositionSequence> merge(List<TextPageBlock> textBlocks) {
+
+        return textBlocks.stream()
+                .map(TextPageBlock::getSequences)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toList());
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionSequenceComparator.java

@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.knecon.fforesight.service.layoutparser.processor.utils;
-
-import java.util.Comparator;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-
-/**
- * This class is a comparator for TextPosition operators.  It handles
- * pages with text in different directions by grouping the text based
- * on direction and sorting in that direction. This allows continuous text
- * in a given direction to be more easily grouped together.
- *
- * @author Ben Litchfield
- */
-public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
-
-    @Override
-    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
-        // only compare text that is in the same direction
-        int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
-        if (cmp1 != 0) {
-            return cmp1;
-        }
-
-        // get the text direction adjusted coordinates
-        double x1 = pos1.getBBox().getX();
-        double x2 = pos2.getBBox().getX();
-
-        double pos1YBottom = pos1.getBBox().getMaxY();
-        double pos2YBottom = pos2.getBBox().getMaxY();
-
-        // note that the coordinates have been adjusted so 0,0 is in upper left
-        double pos1YTop = pos1YBottom - pos1.getBBox().getHeight();
-        double pos2YTop = pos2YBottom - pos2.getBBox().getHeight();
-
-        double yDifference = Math.abs(pos1YBottom - pos2YBottom);
-
-        // we will do a simple tolerance comparison
-        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
-            return Double.compare(x1, x2);
-        } else if (pos1YBottom < pos2YBottom) {
-            return -1;
-        } else {
-            return 1;
-        }
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/visualization/LayoutDebugLayer.java

@@ -1,11 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor.visualization;
 
 import java.awt.Color;
+import java.awt.geom.AffineTransform;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
@@ -14,15 +18,19 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.viewerdoc.layers.LayoutDebugLayerConfig;
 import com.knecon.fforesight.service.viewerdoc.model.ColoredLine;
 import com.knecon.fforesight.service.viewerdoc.model.ColoredRectangle;
+import com.knecon.fforesight.service.viewerdoc.model.FilledRectangle;
 import com.knecon.fforesight.service.viewerdoc.model.PlacedText;
 import com.knecon.fforesight.service.viewerdoc.model.VisualizationsOnPage;
 
@@ -42,6 +50,8 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
 
     boolean active;
 
+    Map<Integer, AtomicInteger> outlineObjectsWithoutPointsPerPage = new HashMap<>();
+
 
     public void addTextVisualizations(List<TextPositionSequence> textPositionSequences, int pageNumber) {
 
@@ -51,7 +61,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(pageNumber, this.words);
         visualizationsOnPage.getColoredRectangles()
                 .addAll(textPositionSequences.stream()
-                                .map(BoundingBox::getBBoxInitialUserSpace)
+                                .map(BoundingBox::getBBoxPdf)
                                 .map(rect -> new ColoredRectangle(rect, WORDS_COLOR, 1))
                                 .toList());
     }
@@ -105,7 +115,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(pageNumber, this.cells);
         visualizationsOnPage.getColoredRectangles()
                 .addAll(cells.stream()
-                                .map(cell -> new ColoredRectangle(cell.getBBoxInitialUserSpace(), CELLS_COLOR, 1))
+                                .map(cell -> new ColoredRectangle(cell.getBBoxPdf(), CELLS_COLOR, 1))
                                 .toList());
     }
 
@@ -119,7 +129,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(page, this.zones);
         visualizationsOnPage.getColoredRectangles()
                 .addAll(zones.stream()
-                                .map(BoundingBox::getBBoxInitialUserSpace)
+                                .map(BoundingBox::getBBoxPdf)
                                 .map(zone -> new ColoredRectangle(zone, ZONES_COLOR, 1))
                                 .toList());
 
@@ -144,12 +154,29 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(page, this.lines);
         visualizationsOnPage.getColoredRectangles()
                 .addAll(lines.stream()
-                                .map(BoundingBox::getBBoxInitialUserSpace)
+                                .map(BoundingBox::getBBoxPdf)
                                 .map(line -> new ColoredRectangle(line, LINES_COLOR, 0.5f))
                                 .toList());
     }
 
 
+    public void addLineVisualizationsFromNestedTextPosition(Collection<Set<TextPositionSequence>> lines, int pageNumber) {
+
+        if (!active) {
+            return;
+        }
+        VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(pageNumber, this.lines);
+        visualizationsOnPage.getColoredRectangles()
+                .addAll(lines.stream()
+                                .map(line -> line.stream()
+                                        .map(BoundingBox::getBBoxPdf)
+                                        .collect(RectangleTransformations.collectBBox()))
+                                .map(line -> new ColoredRectangle(line, LINES_COLOR, 0.5f))
+                                .toList());
+
+    }
+
+
     public void addTextBlockVisualizations(List<TextPageBlock> textPageBlocks, int page) {
 
         if (!active) {
@@ -158,7 +185,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(page, zones);
         visualizationsOnPage.getColoredRectangles()
                 .addAll(textPageBlocks.stream()
-                                .map(rect -> new ColoredRectangle(rect.getBBoxInitialUserSpace(), ZONES_COLOR, 1))
+                                .map(rect -> new ColoredRectangle(rect.getBBoxPdf(), ZONES_COLOR, 1))
                                 .toList());
     }
 
@@ -222,11 +249,11 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
                 .flatMap(Collection::stream)
                 .forEach(character -> {
                     Color color = ROTATING_CHARACTER_COLOR.get(index.getAndIncrement() % ROTATING_CHARACTER_COLOR.size());
-                    Rectangle2D charBBox = character.getTextPosition().getBBoxInitialUserSpace();
+                    Rectangle2D charBBox = character.getTextPosition().getBBoxPdf();
                     characterVisualizations.getColoredRectangles().add(new ColoredRectangle(charBBox, color, 1));
                     character.getNeighbors()
                             .forEach(neighbor -> {
-                                Rectangle2D neighborBBox = neighbor.getCharacter().getTextPosition().getBBoxInitialUserSpace();
+                                Rectangle2D neighborBBox = neighbor.getCharacter().getTextPosition().getBBoxPdf();
                                 Line2D line = new Line2D.Double(new Point2D.Double(charBBox.getCenterX(), charBBox.getCenterY()),
                                                                 new Point2D.Double(neighborBBox.getCenterX(), neighborBBox.getCenterY()));
                                 neighbourVisualizations.getColoredLines().add(new ColoredLine(line, color, 1));
@@ -235,4 +262,41 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
 
     }
 
+
+    public void addOutlineObjects(List<OutlineObject> outlineObjects, PageInformation pageInformation) {
+
+        if (!active) {
+            return;
+        }
+
+        for (OutlineObject outlineObject : outlineObjects) {
+            addOutlineObject(outlineObject, pageInformation);
+        }
+    }
+
+
+    private void addOutlineObject(OutlineObject outlineObject, PageInformation pageInformation) {
+
+        int rectSize = 5;
+
+        Point2D point2D;
+        if (outlineObject.getPoint().isPresent()) {
+            point2D = outlineObject.getPoint().get();
+        } else {
+            int numberOfOutlineObjectsWithoutPoints = outlineObjectsWithoutPointsPerPage.computeIfAbsent(outlineObject.getPageNumber(), a -> new AtomicInteger(0))
+                    .getAndIncrement();
+            point2D = new Point2D.Double(10, 10 + numberOfOutlineObjectsWithoutPoints * (10 + rectSize * 2));
+        }
+
+        Point2D textPoint = new Point2D.Double(point2D.getX() + 2 * rectSize, point2D.getY() + rectSize);
+        AffineTransform pageToUserSpaceTransform = CoordinateTransforms.calculatePageCoordsToInitialUserSpaceCoords(pageInformation);
+        pageToUserSpaceTransform.transform(point2D, point2D);
+        pageToUserSpaceTransform.transform(textPoint, textPoint);
+
+        VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(outlineObject.getPageNumber(), outlineObjects);
+        visualizationsOnPage.getFilledRectangles()
+                .add(new FilledRectangle(new Rectangle2D.Double(point2D.getX() - rectSize, point2D.getY() - rectSize, rectSize * 2, rectSize * 2), OUTLINE_OBJECT_COLOR, 1));
+        visualizationsOnPage.getPlacedTexts().add(PlacedText.textFacingUp(outlineObject.getTitle(), textPoint, 10, outlineObject.isFound() ? Color.BLACK : Color.RED, FONT));
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/visualization/LayoutGrid.java

@@ -15,6 +15,7 @@ import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
@@ -72,6 +73,9 @@ public class LayoutGrid extends LayoutGridLayerConfig {
     public void addHeadline(Headline headline) {
 
         addAsRectangle(headline, headlines, HEADLINE_COLOR);
+        if (headline.getEngines().contains(LayoutEngine.OUTLINE)) {
+            addAsRectangle(headline, outlineHeadlines, HEADLINE_COLOR);
+        }
     }
 
 
@@ -129,9 +133,6 @@ public class LayoutGrid extends LayoutGridLayerConfig {
                 .collect(Collectors.toList());
         pagesInOrder.remove(0);
         handleFirstPageOfSection(section, firstPage, bBoxMap.get(firstPage), treeIdString, maxChildDepth, ownDepth);
-        if (section instanceof SuperSection) {
-            return;
-        }
         for (Page middlePage : pagesInOrder.subList(0, pagesInOrder.size() - 1)) {
             handleForMiddlePageOfSection(section, middlePage, bBoxMap.get(middlePage), treeIdString, maxChildDepth, ownDepth);
         }
@@ -199,9 +200,9 @@ public class LayoutGrid extends LayoutGridLayerConfig {
         List<PlacedText> placedTexts = getOrCreateVisualizationsOnPage(page.getNumber(), visualizations).getPlacedTexts();
 
         PlacedText newText = PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, color, FONT);
-
+        float threshold = 1.5f * FONT_SIZE;
         Optional<PlacedText> conflictingText = placedTexts.stream()
-                .filter(pt -> Math.abs(pt.lineStart().getY() - newText.lineStart().getY()) <= FONT_SIZE)
+                .filter(pt -> Math.abs(pt.lineStart().getY() - newText.lineStart().getY()) <= threshold || Math.abs(pt.lineStart().getX() - newText.lineStart().getX()) <= threshold)
                 .findFirst();
 
         if (conflictingText.isPresent()) {

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -30,7 +30,7 @@ dependencies {
     implementation(project(":layoutparser-service-internal-api"))
 
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.30.0")
     implementation("com.knecon.fforesight:tracing-commons:0.5.0")
     implementation("com.knecon.fforesight:lifecycle-commons:0.6.0")
     implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")
@@ -38,7 +38,7 @@ dependencies {
     implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
-    implementation("com.pdftron:PDFNet:10.7.0")
+    implementation("com.pdftron:PDFNet:10.11.0")
 
     // for integration testing only
     testImplementation(project(":viewer-doc-processor"))
@@ -52,6 +52,8 @@ dependencies {
     testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    testImplementation("org.apache.commons:commons-text:1.12.0")
+
 }
 
 /*
@@ -89,6 +91,9 @@ tasks.named<BootBuildImage>("bootBuildImage") {
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
     environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
 
+    builder.set("docker-proxy.knecon.com/paketobuildpacks/builder:base")
+    runImage.set("docker-proxy.knecon.com/paketobuildpacks/run:base-cnb")
+
     imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
     if (project.hasProperty("buildbootDockerHostNetwork")) {
         network.set("host")
@@ -99,6 +104,13 @@ tasks.named<BootBuildImage>("bootBuildImage") {
         }
         verboseLogging.set(true)
 
+        builderRegistry {
+            username.set(providers.gradleProperty("mavenUser").getOrNull())
+            password.set(providers.gradleProperty("mavenPassword").getOrNull())
+            email.set(providers.gradleProperty("mavenEmail").getOrNull())
+            url.set("https://docker-proxy.knecon.com:5001/")
+        }
+
         publishRegistry {
             username.set(providers.gradleProperty("mavenUser").getOrNull())
             password.set(providers.gradleProperty("mavenPassword").getOrNull())
@@ -106,4 +118,5 @@ tasks.named<BootBuildImage>("bootBuildImage") {
             url.set("https://nexus.knecon.com:5001/")
         }
     }
+
 }

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/PDFNetInitializer.java

@@ -1,20 +1,18 @@
 package com.knecon.fforesight.service.layoutparser.server;
 
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
 
 import com.google.common.base.Strings;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutparserSettings;
 import com.pdftron.pdf.PDFNet;
 
-import jakarta.annotation.PostConstruct;
-import jakarta.annotation.PreDestroy;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-@Component
+@Configuration
 @RequiredArgsConstructor
 public class PDFNetInitializer {
 
@@ -22,26 +20,17 @@ public class PDFNetInitializer {
     private String pdftronLicense;
 
 
+    @Bean
     @SneakyThrows
-    @PostConstruct
-    // Do not change back to application runner, if it is application runner it takes messages from the queue before PDFNet is initialized, that leads to UnsatisfiedLinkError.
     public void init() {
 
         if (Strings.isNullOrEmpty(pdftronLicense)) {
-            return;
+            throw new IllegalArgumentException("PDFTRON_LICENSE not set!");
         }
         log.info("Initializing Native Libraries");
         log.info("Setting pdftron license: {}", pdftronLicense);
         PDFNet.setTempPath("/tmp/pdftron");
         PDFNet.initialize(pdftronLicense);
-
-    }
-
-
-    @PreDestroy
-    public void terminate() {
-
-        PDFNet.terminate();
     }
 
 }

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/configuration/MessagingConfiguration.java

@@ -0,0 +1,29 @@
+package com.knecon.fforesight.service.layoutparser.server.configuration;
+
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
+
+import org.springframework.amqp.core.DirectExchange;
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+
+@Configuration
+public class MessagingConfiguration {
+
+    @Bean
+    public DirectExchange layoutParsingRequestExchange() {
+
+        return new DirectExchange(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue layoutParsingDLQ() {
+
+        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/configuration/TenantMessagingConfigurationImpl.java

@@ -0,0 +1,11 @@
+package com.knecon.fforesight.service.layoutparser.server.configuration;
+
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
+
+@Configuration
+public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
+
+
+}

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -18,6 +18,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -28,15 +29,17 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class MessageHandler {
 
+    public static final String LAYOUT_PARSING_REQUEST_LISTENER_ID = "layout-parsing-request-listener";
+
     private final LayoutParsingPipeline layoutParsingPipeline;
     private final ObjectMapper objectMapper;
     private final RabbitTemplate rabbitTemplate;
     private final static String X_PIPELINE_PREFIX = "X-PIPE-";
 
 
-    @RabbitHandler
-    @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)
     @SneakyThrows
+    @RabbitHandler
+    @RabbitListener(id = LAYOUT_PARSING_REQUEST_LISTENER_ID)
     public void receiveLayoutParsingRequest(Message message) {
 
         LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);
@@ -57,7 +60,7 @@ public class MessageHandler {
     public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent, Message message) {
 
         Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent, m -> {
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE, TenantContext.getTenantId(), layoutParsingFinishedEvent, m -> {
             var forwardHeaders = message.getMessageProperties()
                     .getHeaders()
                     .entrySet()

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/TenantExchangeMessageReceiverImpl.java

@@ -0,0 +1,64 @@
+package com.knecon.fforesight.service.layoutparser.server.queue;
+
+import java.util.Set;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+import com.knecon.fforesight.tenantcommons.TenantProvider;
+import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
+import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
+import com.knecon.fforesight.tenantcommons.model.TenantResponse;
+import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
+import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
+
+@Service
+public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
+
+    public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
+
+        super(rabbitQueueService, tenantProvider);
+    }
+
+
+    @Override
+    protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
+
+        return Set.of(TenantQueueConfiguration.builder()
+                              .listenerId(MessageHandler.LAYOUT_PARSING_REQUEST_LISTENER_ID)
+                              .exchangeName(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE)
+                              .queuePrefix(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE_PREFIX)
+                              .dlqName(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ)
+                              .build());
+    }
+
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void onApplicationReady() {
+
+        System.out.println("application ready invoked");
+        super.initializeQueues();
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
+    public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
+
+        super.reactToTenantCreation(tenantCreatedEvent);
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
+    public void reactToTenantDeletion(TenantResponse tenantResponse) {
+
+        super.reactToTenantDeletion(tenantResponse);
+
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/main/resources/logback-spring.xml

@@ -14,4 +14,6 @@
         <appender-ref ref="${logType}"/>
     </root>
 
+    <logger name="org.apache.fontbox.ttf" level="ERROR"/>
+
 </configuration>

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/processor/model/SectionIdentifierTest.java

@@ -0,0 +1,86 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+class SectionIdentifierTest {
+
+    @Test
+    void testSectionIdentifier() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("1.1.2: Headline");
+        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
+        assertEquals(3, identifier.level());
+        assertEquals(List.of(1, 1, 2), identifier.getIdentifiers());
+
+        SectionIdentifier child = SectionIdentifier.asChildOf(identifier);
+        assertTrue(child.isChildOf(identifier));
+
+        SectionIdentifier parent = SectionIdentifier.fromSearchText("1.1: Headline");
+        assertTrue(parent.isParentOf(identifier));
+    }
+
+
+    @Test
+    void testSectionIdentifier2() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("A.1.2: Headline");
+        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
+        assertEquals(3, identifier.level());
+        assertEquals(List.of(1, 1, 2), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier3() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("D.1.2: Headline");
+        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
+        assertEquals(3, identifier.level());
+        assertEquals(List.of(4, 1, 2), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier4() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("4.1.2.4: Headline");
+        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
+        assertEquals(4, identifier.level());
+        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier5() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("D.1.2.4.5: Headline");
+        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
+        assertEquals(4, identifier.level());
+        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier6() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("d.1.2.4.5: Headline");
+        assertEquals(SectionIdentifier.Format.ALPHANUMERIC, identifier.getFormat());
+        assertEquals(4, identifier.level());
+        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
+    }
+
+
+    @Test
+    void testSectionIdentifier7() {
+
+        SectionIdentifier identifier = SectionIdentifier.fromSearchText("4.1.2.4.5: Headline");
+        assertEquals(SectionIdentifier.Format.NUMERICAL, identifier.getFormat());
+        assertEquals(4, identifier.level());
+        assertEquals(List.of(4, 1, 2, 4), identifier.getIdentifiers());
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -10,7 +10,9 @@ import java.util.stream.Collectors;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -55,6 +57,12 @@ public class HeadlinesGoldStandardIntegrationTest {
     @MockBean
     private RabbitTemplate rabbitTemplate;
 
+    @MockBean
+    private RabbitAdmin rabbitAdmin;
+
+    @MockBean
+    private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
+
     @Autowired
     private ObjectMapper objectMapper;
 
@@ -70,13 +78,18 @@ public class HeadlinesGoldStandardIntegrationTest {
 
         List<Metrics> metrics = new ArrayList<>();
         metrics.add(getMetrics("files/syngenta/CustomerFiles/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf",
-                "files/headlineTest/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1)_REDACTION_LOG.json"));
+                               "files/headlineTest/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1)_REDACTION_LOG.json"));
         metrics.add(getMetrics("files/syngenta/CustomerFiles/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf",
-                "files/headlineTest/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23_REDACTION_LOG.json"));
-        metrics.add(getMetrics("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", "files/headlineTest/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
+                               "files/headlineTest/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23_REDACTION_LOG.json"));
+        metrics.add(getMetrics("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf",
+                               "files/headlineTest/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
 
-        double precision = metrics.stream().mapToDouble(Metrics::getPrecision).average().orElse(1.0);
-        double recall = metrics.stream().mapToDouble(Metrics::getRecall).average().orElse(1.0);
+        double precision = metrics.stream()
+                .mapToDouble(Metrics::getPrecision).average()
+                .orElse(1.0);
+        double recall = metrics.stream()
+                .mapToDouble(Metrics::getRecall).average()
+                .orElse(1.0);
 
         System.out.println("Precision is: " + precision + " recall is: " + recall);
 
@@ -93,21 +106,28 @@ public class HeadlinesGoldStandardIntegrationTest {
 
         Set<Headline> goldStandardHeadlines = new HashSet<>();
         var goldStandardLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class);
-        goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
-        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));
+        goldStandardLog.getRedactionLogEntry()
+                .removeIf(r -> !r.isRedacted() || r.getChanges()
+                        .get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
+        goldStandardLog.getRedactionLogEntry()
+                .forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions()
+                                                                             .get(0).getPage(), e.getValue())));
 
         Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE,
                                                                          layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
-                        pdfFileResource.getFile(),
-                        new ImageServiceResponse(),
-                        new TableServiceResponse(),
-                        new VisualLayoutParsingResponse(),
-                        Map.of("file",filePath)));
+                                                                                                           pdfFileResource.getFile(),
+                                                                                                           new ImageServiceResponse(),
+                                                                                                           new TableServiceResponse(),
+                                                                                                           new VisualLayoutParsingResponse(),
+                                                                                                           Map.of("file", filePath)));
 
         var foundHeadlines = documentGraph.streamAllSubNodes()
                 .map(SemanticNode::getHeadline)
                 .distinct()
-                .map(headlineNode -> new Headline(headlineNode.getPages().stream().findFirst().get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing()))
+                .map(headlineNode -> new Headline(headlineNode.getPages()
+                                                          .stream()
+                                                          .findFirst()
+                                                          .get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing()))
                 .toList();
 
         Set<Headline> correct = new HashSet<>();
@@ -121,7 +141,9 @@ public class HeadlinesGoldStandardIntegrationTest {
             }
         }
 
-        missing = goldStandardHeadlines.stream().filter(h -> !correct.contains(h)).collect(Collectors.toSet());
+        missing = goldStandardHeadlines.stream()
+                .filter(h -> !correct.contains(h))
+                .collect(Collectors.toSet());
 
         float precision = (float) correct.size() / (float) foundHeadlines.size();
         float recall = (float) correct.size() / ((float) correct.size() + (float) missing.size());

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java

@@ -27,24 +27,28 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 public class LayoutparserEnd2EndTest extends AbstractTest {
 
+    public static final LayoutParsingType LAYOUT_PARSING_TYPE = LayoutParsingType.DOCUMINE_OLD;
+
     @Autowired
     private LayoutParsingPipeline layoutParsingPipeline;
 
+
     @Test
     @Disabled
     public void testLayoutParserEndToEnd() {
 
-        String filePath = "files/syngenta/CustomerFiles/Documine/Flora/425_F.1.1.1 - A13617AV - Acute Oral Toxicity Study.pdf";
+        String filePath = "/home/kschuettler/Dokumente/TestFiles/NER Dataset/Syngenta prod/77c680315c31d403d2e023be023b2087.PREVIEW.pdf";
 
         runForFile(filePath);
     }
 
+
     @Test
     @Disabled
     @SneakyThrows
     public void testLayoutParserEndToEndWithFolder() {
 
-        String folder = "/Users/maverickstuder/Documents/Fforesight/layoutparser/layoutparser-service/layoutparser-service-server/src/test";
+        String folder = "/home/kschuettler/Dokumente/Ticket Related/RED-9975";
         List<Path> pdfFiles = Files.walk(Path.of(folder))
                 .filter(path -> path.getFileName().toString().endsWith(".pdf"))
                 .sorted(Comparator.comparing(Path::getFileName))
@@ -70,7 +74,8 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
             file = new File(filePath);
         }
 
-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, true);
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(fileName, LAYOUT_PARSING_TYPE, true);
+
         prepareStorage(layoutParsingRequest, file);
 
         LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/OutlineDetectionTest.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.server;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -35,7 +36,6 @@ import com.knecon.fforesight.service.layoutparser.processor.services.visualizati
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;
 
-import jakarta.annotation.PostConstruct;
 import lombok.SneakyThrows;
 
 public class OutlineDetectionTest extends AbstractTest {
@@ -57,6 +57,17 @@ public class OutlineDetectionTest extends AbstractTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    public void testOutlineError() {
+
+        String fileName = "files/syngenta/CustomerFiles/Clarifynd/VV-470942.pdf";
+
+        ClassificationDocument classificationDocument = parseLayout(fileName, LayoutParsingType.CLARIFYND);
+        assertThat(classificationDocument).isNotNull();
+    }
+
+
     @Test
     @SneakyThrows
     public void testOutlinesToSections() {
@@ -69,20 +80,21 @@ public class OutlineDetectionTest extends AbstractTest {
 
         long start = System.currentTimeMillis();
         ClassificationDocument classificationDocument = parseLayout(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
-
+        Document document = buildGraph(fileName, classificationDocument);
+        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
         OutlineObjectTree outlineObjectTree = classificationDocument.getOutlineObjectTree();
         assertEquals(outlineObjectTree.getRootNodes().size(), 8);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(1).size(), 1);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(3).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(2).size(), 1);
         assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(4).size(), 1);
         assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(5).size(), 1);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(6).size(), 2);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(7).size(), 3);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(8).size(), 2);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(10).size(), 1);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(11).size(), 4);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(12).size(), 1);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(13).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(6).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(7).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(8).size(), 3);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(9).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(11).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(12).size(), 4);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(13).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(14).size(), 2);
         assertTrue(outlineObjectTree.getOutlineObjectsPerPage().values()
                            .stream()
                            .flatMap(Collection::stream)
@@ -90,7 +102,7 @@ public class OutlineDetectionTest extends AbstractTest {
 
         TableOfContents tableOfContents = classificationDocument.getTableOfContents();
 
-        assertEquals(tableOfContents.getMainSections().size(), 9);
+        assertEquals(tableOfContents.getMainSections().size(), 10);
         assertEquals(tableOfContents.getMainSections().subList(1, 9)
                              .stream()
                              .map(tableOfContentItem -> sanitizeString(tableOfContentItem.getHeadline().toString()))
@@ -99,17 +111,15 @@ public class OutlineDetectionTest extends AbstractTest {
                              .stream()
                              .map(outlineObjectTreeNode -> sanitizeString(outlineObjectTreeNode.getOutlineObject().getTitle()))
                              .toList());
-        assertEquals(tableOfContents.getMainSections().get(5).getChildren().size(), 6);
-        assertEquals(tableOfContents.getMainSections().get(7).getChildren().size(), 3);
-        assertEquals(tableOfContents.getMainSections().get(8).getChildren().size(), 3);
-        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().size(), 1);
-        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().get(0).getChildren().size(), 3);
-
-        assertEquals(tableOfContents.getMainSections().get(0).getImages().size(), 1);
-        assertEquals(tableOfContents.getMainSections().get(6).getImages().size(), 1);
-        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().get(0).getChildren().get(2).getImages().size(), 1);
-
-        Document document = buildGraph(fileName, classificationDocument);
+//        assertEquals(tableOfContents.getMainSections().get(5).getChildren().size(), 6);
+//        assertEquals(tableOfContents.getMainSections().get(7).getChildren().size(), 3);
+//        assertEquals(tableOfContents.getMainSections().get(8).getChildren().size(), 3);
+//        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().size(), 1);
+//        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().get(0).getChildren().size(), 3);
+//
+//        assertEquals(tableOfContents.getMainSections().get(0).getImages().size(), 1);
+//        assertEquals(tableOfContents.getMainSections().get(6).getImages().size(), 1);
+//        assertEquals(tableOfContents.getMainSections().get(8).getChildren().get(2).getChildren().get(0).getChildren().get(2).getImages().size(), 1);
 
         assertTrue(tableOfContents.getAllTableOfContentItems()
                            .stream()
@@ -125,7 +135,7 @@ public class OutlineDetectionTest extends AbstractTest {
 
         List<SemanticNode> childrenOfTypeSectionOrSuperSection = document.getChildrenOfTypeSectionOrSuperSection();
 
-        assertEquals(childrenOfTypeSectionOrSuperSection.size(), 9);
+        assertEquals(childrenOfTypeSectionOrSuperSection.size(), 10);
         assertEquals(childrenOfTypeSectionOrSuperSection.subList(1, 9)
                              .stream()
                              .map(section -> sanitizeString(section.getHeadline().getLeafTextBlock().toString()))
@@ -134,38 +144,37 @@ public class OutlineDetectionTest extends AbstractTest {
                              .stream()
                              .map(outlineObjectTreeNode -> sanitizeString(outlineObjectTreeNode.getOutlineObject().getTitle()))
                              .toList());
-        Predicate<SemanticNode> isSectionOrSuperSection = semanticNode -> semanticNode instanceof Section || semanticNode instanceof SuperSection;
-        assertEquals(childrenOfTypeSectionOrSuperSection.get(5).streamChildren()
-                             .filter(isSectionOrSuperSection)
-                             .count(), 6 + 1); // 1 additional for main text of parent section
-        assertEquals(childrenOfTypeSectionOrSuperSection.get(7).streamChildren()
-                             .filter(isSectionOrSuperSection)
-                             .count(), 3 + 1);
-        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
-                             .filter(isSectionOrSuperSection)
-                             .count(), 3 + 1);
-        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
-                             .filter(isSectionOrSuperSection)
-                             .toList().get(3).streamChildren()
-                             .filter(isSectionOrSuperSection)
-                             .count(), 1 + 1);
-        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
-                             .filter(isSectionOrSuperSection)
-                             .toList().get(3).streamChildren()
-                             .filter(isSectionOrSuperSection)
-                             .toList().get(1).streamChildren()
-                             .filter(isSectionOrSuperSection)
-                             .count(), 3 + 1);
+//        Predicate<SemanticNode> isSectionOrSuperSection = semanticNode -> semanticNode instanceof Section || semanticNode instanceof SuperSection;
+//        assertEquals(childrenOfTypeSectionOrSuperSection.get(5).streamChildren()
+//                             .filter(isSectionOrSuperSection)
+//                             .count(), 6 + 1); // 1 additional for main text of parent section
+//        assertEquals(childrenOfTypeSectionOrSuperSection.get(7).streamChildren()
+//                             .filter(isSectionOrSuperSection)
+//                             .count(), 3 + 1);
+//        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+//                             .filter(isSectionOrSuperSection)
+//                             .count(), 3 + 1);
+//        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+//                             .filter(isSectionOrSuperSection)
+//                             .toList().get(3).streamChildren()
+//                             .filter(isSectionOrSuperSection)
+//                             .count(), 1 + 1);
+//        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+//                             .filter(isSectionOrSuperSection)
+//                             .toList().get(3).streamChildren()
+//                             .filter(isSectionOrSuperSection)
+//                             .toList().get(1).streamChildren()
+//                             .filter(isSectionOrSuperSection)
+//                             .count(), 3 + 1);
 
-        List<List<Integer>> imageTreeIdList = document.streamAllImages()
-                .map(image -> image.getParent().getTreeId())
-                .toList();
+//        List<List<Integer>> imageTreeIdList = document.streamAllImages()
+//                .map(image -> image.getParent().getTreeId())
+//                .toList();
+//
+//        assertEquals(imageTreeIdList.get(0), List.of(0));
+//        assertEquals(imageTreeIdList.get(1), List.of(6));
+//        assertEquals(imageTreeIdList.get(2), List.of(8, 4, 2, 4));
 
-        assertEquals(imageTreeIdList.get(0), List.of(0));
-        assertEquals(imageTreeIdList.get(1), List.of(6));
-        assertEquals(imageTreeIdList.get(2), List.of(8, 4, 2, 4));
-
-        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentReadingOrderTest.java

@@ -0,0 +1,465 @@
+package com.knecon.fforesight.service.layoutparser.server.graph;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
+import java.io.FileOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.text.similarity.LevenshteinDistance;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
+import com.knecon.fforesight.service.layoutparser.server.PDFNetInitializer;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
+import com.knecon.fforesight.service.viewerdoc.model.Standard14EmbeddableFont;
+import com.pdftron.common.Matrix2D;
+import com.pdftron.common.PDFNetException;
+import com.pdftron.pdf.ColorPt;
+import com.pdftron.pdf.ColorSpace;
+import com.pdftron.pdf.Element;
+import com.pdftron.pdf.ElementBuilder;
+import com.pdftron.pdf.ElementWriter;
+import com.pdftron.pdf.Font;
+import com.pdftron.pdf.GState;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.sdf.SDFDoc;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DocumentReadingOrderTest extends BuildDocumentTest {
+
+    private static final boolean DRAW_DIR_ADJ_COORDS = true;
+    public static final List<LayoutParsingType> LAYOUT_PARSING_TYPES = List.of(LayoutParsingType.DOCUMINE,
+                                                                               LayoutParsingType.DOCUMINE_OLD,
+                                                                               LayoutParsingType.REDACT_MANAGER,
+                                                                               LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
+
+    @Autowired
+    PDFNetInitializer pdfNetInitializer;
+
+    @Autowired
+    StorageService storageService;
+
+
+    @BeforeEach
+    public void before() {
+
+        pdfNetInitializer.init();
+    }
+
+
+    @AfterEach
+    public void cleanUp() {
+
+        ((FileSystemBackedStorageService) storageService).clearStorage();
+    }
+
+
+    @Test
+    @Disabled
+    public void drawDirAdjForFile() {
+
+        String pdfFile = "/home/kschuettler/Dokumente/Ticket Related/RED-9974/026dc94b019bc2348a4c54f0c6c4516f.ORIGIN.pdf";
+
+        ClassificationDocument classificationDocument = parseLayout(pdfFile, LayoutParsingType.DOCUMINE_OLD);
+
+        drawDirAdjCoords(pdfFile, classificationDocument, LayoutParsingType.DOCUMINE_OLD);
+    }
+
+
+    @Test
+    public void readingOrderTestSeite14() {
+
+        String pdfFile = "files/syngenta/CustomerFiles/SinglePages/Seite14.pdf";
+        String expectedText = """
+                27
+                26 APPENDICES SECTION
+                APPENDIX 1 Analytical Report
+                syngenta
+                A16148F
+                Batch ID 533158 (GP-080305)
+                Batch Identification 533158
+                Product Design Code A16148F
+                Product Denomination SYN524464 FS (500)
+                Product by Common Name SYN524464 FS (500)
+                Other Product Code(s) GP-080305
+                Source Technology & Projects, Syngenta Crop Protection, Inc.
+                Chemical Analysis
+                (Active Ingredient Content)
+                Identity of the Active Ingredient* Confirmed
+                Content of SYN524464* 45.6% (wt/wt) or 534 g/L
+                Methodology Used for Characterization HPLC
+                The Active Ingredient content is within the FAO limits.
+                Physical Analysis
+                Appearance* pink opaque liquid
+                Density* 1171 g/L
+                Stability:
+                Storage Temperature <30°℃
+                Expiration date March 2009
+                The stability of this test substance will be determined concurrently through reanalysis of material held
+                in inventory under GLP conditions at Syngenta Crop Protection, Inc., Greensboro, NC
+                This Certificate of Analysis is summarizing data (marked with an asterisk) from a study that has been
+                performed in compliance with Good Laboratory Practices per 40 CFR Part 160 Raw data,
+                documentation, protocols, any amendments to study protocols and reports pertaining to this study are
+                maintained in the Syngenta Crop Protection Archives in Greensboro, NC.
+                Authorization'
+                26 Mar 2008
+                Dorothea Jeffery Date
+                Group Leader I
+                Analytical & Product Chemistry Department
+                Document 10350420.doc Certificate of Analysis
+                Page 1 of 1 Study T000973-08
+                Report Number: 11813-08 Page 14 of 14
+                """;
+
+        assertSimilarReadingOrder(expectedText, pdfFile);
+    }
+
+
+    @Test
+    public void readingOrderTestTiltedText() {
+
+        String pdfFile = "files/syngenta/CustomerFiles/SinglePages/tiltedText.pdf";
+        String expectedText = """
+                However there was no consistency in the areas affected either between sexes or at different
+                ages, in general other measurements for the same structures at other levels showed no
+                differences, all were within the historical control range of mean values and none of these
+                differences is considered to be related to treatment (Appendix K).
+                7. DISCUSSION
+                The purpose of this study, which was to determine the potential for developmental
+                neurotoxicity in the assessment and evaluation of the toxic characteristics of lambda-
+                cyhalothrin in rats, was successfully accomplished.
+                There was evidence of toxicity characterised by lower bodyweights and food consumption in
+                dams receiving 60 or 150 ppm lambda-cyhalothrin during gestation and also post partum in
+                the 150 ppm group only.
+                There were no treatment-related effects of administration of lambda-cyhalothrin on
+                reproductive parameters: there were no effects on gestation length, mean litter size or on pup
+                bodyweight at birth.
+                There was evidence of toxicity in F1 animals receiving 150 ppm. This was seen as slightly
+                higher pup mortality up to day 5 and lower bodyweights from day 5, reaching a maximum of
+                approximately 8-9% below control on day 22.
+                There was a small difference in the age at which male rats in the 150 ppm group reached
+                preputial separation, but this was too small to be of toxicological significance.
+                No effects were seen on motor activity or response to auditory startle.
+                There was no clear evidence of any effects in the learning and memory assessment in
+                weanling (age 21-24 days) or young adult animals (age 59-62 days). However, at day 21
+                swimming speeds of females receiving 150 ppm were slightly slower than controls. The
+                difference is considered to reflect a difference in swimming performance rather than an effect
+                on learning or memory.
+                No neuropathological effect of treatment with lambda-cyhalothrin was detected from a
+                detailed microscopic examination of the selected F1 animals post mortem on day 12 or 63.
+                LAMBDA-CYHALOTHRIN: DEVELOPMENTAL NEUROTOXICITY STUDY IN RATS
+                CTL/RR0969/REGULATORY/REPORT - 34
+                """;
+
+        assertSimilarReadingOrder(expectedText, pdfFile);
+    }
+
+
+    @Test
+    public void readingOrderTest402Study() {
+
+        String pdfFile = "files/SinglePages/402StudyPage5.pdf";
+        String expectedText = """
+                2.0 INTRODUCTION
+                2.1 Purpose
+                Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
+                invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et
+                accusam et justo duo dolores et ea rebum.
+                Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem
+                ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
+                ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et
+                justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est
+                Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
+                diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
+                voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd
+                gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+                2.2 Guidelines
+                Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
+                invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.
+                At vero eos et accusam et justo duo dolores et ea rebum.
+                Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem
+                ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
+                ut labore et dolore magna aliquyam erat, sed diam voluptua.
+                At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no
+                sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet,
+                consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore
+                magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et
+                ea rebum.
+                Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+                Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel
+                illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui
+                blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem
+                ipsum dolor sit amet.
+                2.3 Test Facility
+                Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
+                invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et
+                accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata
+                sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur
+                sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna
+                aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea
+                rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+                Report Number: 20/080-002P 5
+                """;
+
+        assertSimilarReadingOrder(expectedText, pdfFile);
+    }
+
+
+    @Test
+    public void readingOrderTest402StudyRotated() {
+
+        String pdfFile = "files/SinglePages/402StudyPage5_rotated.pdf";
+        String expectedText = """
+                2.0 INTRODUCTION
+                2.1 Purpose
+                Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
+                invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et
+                accusam et justo duo dolores et ea rebum.
+                Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem
+                ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
+                ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et
+                justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est
+                Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
+                diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
+                voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd
+                gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+                2.2 Guidelines
+                Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
+                invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.
+                At vero eos et accusam et justo duo dolores et ea rebum.
+                Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem
+                ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
+                ut labore et dolore magna aliquyam erat, sed diam voluptua.
+                At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no
+                sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet,
+                consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore
+                magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et
+                ea rebum.
+                Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+                Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel
+                illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui
+                blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem
+                ipsum dolor sit amet.
+                2.3 Test Facility
+                Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
+                invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et
+                accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata
+                sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur
+                sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna
+                aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea
+                rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+                Report Number: 20/080-002P 5
+                """;
+
+        assertSimilarReadingOrder(expectedText, pdfFile);
+    }
+
+
+    private void assertSimilarReadingOrder(String expectedText, String pdfFile) {
+
+        List<String> expectedLines = List.of(expectedText.split("\n"));
+        for (LayoutParsingType layoutParsingType : LAYOUT_PARSING_TYPES) {
+
+            log.info("Evaluating for {}", layoutParsingType);
+
+            ClassificationDocument classificationDocument = parseLayout(pdfFile, layoutParsingType);
+
+            if (DRAW_DIR_ADJ_COORDS) {
+                drawDirAdjCoords(pdfFile, classificationDocument, layoutParsingType);
+            }
+
+            Document document = DocumentGraphFactory.buildDocumentGraph(layoutParsingType, classificationDocument);
+            List<String> readLines = getTextAsLines(document);
+            readLines.forEach(log::info);
+
+            int correctCount = 0;
+            int maxLineOffset = 0;
+            for (int i = 0; i < expectedLines.size(); i++) {
+                String expectedLine = expectedLines.get(i);
+                int mostSimilarLine = 0;
+                double maxSimilarity = 0;
+                for (int j = 0; j < readLines.size(); j++) {
+                    String readLine = readLines.get(j);
+                    double similarity = similarity(expectedLine, readLine);
+                    if (similarity > maxSimilarity) {
+                        maxSimilarity = similarity;
+                        mostSimilarLine = j;
+                    }
+                }
+                if (readLines.get(mostSimilarLine).trim().equals(expectedLine.trim())) {
+                    correctCount++;
+                    int lineOffset = Math.abs(mostSimilarLine - i);
+                    if (lineOffset > 0) {
+                        log.info("Line {} offset by {}", readLines.get(mostSimilarLine), lineOffset);
+                    }
+                    if (lineOffset > maxLineOffset) {
+                        maxLineOffset = lineOffset;
+                    }
+                } else {
+                    log.error("Lines {}-{} do not match: \n Expected: {}\n Actual: {}", i, mostSimilarLine, expectedLine, readLines.get(mostSimilarLine));
+                }
+            }
+            double correctLinesFactor = (double) correctCount / (double) readLines.size();
+            double averageLineOffset = (double) maxLineOffset / (double) readLines.size();
+
+            log.info("Difference in number of lines: {}", Math.abs(expectedLines.size() - readLines.size()));
+            log.info("Correct lines factor: {}", correctLinesFactor);
+            log.info("Max order offset: {}, avg: {}", maxLineOffset, averageLineOffset);
+            // In the rotated document one line is read as two
+
+            assertTrue(Math.abs(expectedLines.size() - readLines.size()) <= 1);
+            // Most of the errors come from the similarity metric finding different lines in 402 study, as the lines are too similar, or a miss classification of Footers
+            assertTrue(averageLineOffset < 1);
+            assertTrue(correctLinesFactor > 0.9);
+        }
+    }
+
+
+    public List<String> getTextAsLines(Document document) {
+
+        return document.getTextBlock().getAtomicTextBlocks()
+                .stream()
+                .filter(atb -> !atb.isEmpty())
+                .map(DocumentReadingOrderTest::getLines)
+                .flatMap(List::stream)
+                .toList();
+    }
+
+
+    private static List<String> getLines(AtomicTextBlock atomicTextBlock) {
+
+        int numberOfLines = atomicTextBlock.numberOfLines();
+        List<String> lines = new ArrayList<>(numberOfLines);
+        for (int line = 0; line < numberOfLines; line++) {
+            lines.add(atomicTextBlock.getLine(line).toString());
+        }
+        return lines;
+    }
+
+
+    private static double similarity(String s1, String s2) {
+
+        LevenshteinDistance levenshteinDistance = new LevenshteinDistance();
+
+        int max = Math.max(s1.length(), s2.length());
+        int dist = levenshteinDistance.apply(s1, s2);
+        return 1 - (double) dist / (double) max;
+    }
+
+
+    @SneakyThrows
+    private void drawDirAdjCoords(String filename, ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {
+
+        try (PDFDoc pdfDoc = new PDFDoc(); ElementWriter writer = new ElementWriter(); ElementBuilder builder = new ElementBuilder()) {
+
+            Standard14EmbeddableFont font = Standard14EmbeddableFont.helvetica();
+            Font helvetica = Font.create(pdfDoc, Font.e_helvetica);
+            for (ClassificationPage classificationDocumentPage : classificationDocument.getPages()) {
+                int count = 0;
+                Page page = pdfDoc.pageCreate();
+                writer.begin(page);
+                for (AbstractPageBlock abstractBlock : classificationDocumentPage.getTextBlocks()) {
+
+                    if (abstractBlock instanceof TextPageBlock textBlock) {
+                        for (TextPositionSequence sequence : TextPositionOperations.mergeAndSort(List.of(textBlock))) {
+
+                            float stringWidth;
+                            try {
+                                stringWidth = font.getStringWidth(sequence.toString());
+                            } catch (Exception e) {
+                                stringWidth = font.getFont().getAverageFontWidth() * sequence.toString().length();
+                            }
+                            double fontSize = (sequence.getBBoxDirAdj().getWidth() / stringWidth) * 1000;
+                            try (Matrix2D textMatrix = new Matrix2D(1,
+                                                                    0,
+                                                                    0,
+                                                                    1,
+                                                                    sequence.getXDirAdj(),
+                                                                    page.getCropBox().getHeight() - sequence.getYDirAdj() - sequence.getHeightDirAdj())) {
+                                writeText(sequence.toString(), textMatrix, builder, helvetica, fontSize, writer, Color.BLACK);
+                                writeText(String.valueOf(count), textMatrix.translate(-(2 + (5 * String.valueOf(count).length())), 0), builder, helvetica, 8, writer, Color.RED);
+                                count++;
+                            }
+
+                            writeBBox(sequence.getBBoxDirAdj(), builder, page, writer, Color.BLACK);
+                        }
+                        writeBBox(textBlock.getBBoxDirAdj(), builder, page, writer, Color.BLUE);
+                    }
+                }
+                writer.end();
+                pdfDoc.pagePushBack(page);
+
+            }
+
+            Path stem = Path.of("/tmp/READING_ORDER_TEST/");
+            Files.createDirectories(stem);
+            try (var out = new FileOutputStream(stem.resolve(layoutParsingType.name() + "_" + Path.of(filename).getFileName()).toFile() + "_dirAdjCoordinates.pdf")) {
+                pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
+            }
+        }
+    }
+
+
+    private static void writeBBox(Rectangle2D r, ElementBuilder builder, Page page, ElementWriter writer, Color color) throws PDFNetException {
+
+        Element rect = builder.createRect(r.getX(), page.getCropBox().getHeight() - r.getY(), r.getWidth(), -r.getHeight());
+        float[] comp = color.getColorComponents(null);
+        rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+        try (ColorPt colorpt = new ColorPt(comp[0], comp[1], comp[2])) {
+            rect.getGState().setStrokeColor(colorpt);
+        }
+        rect.setPathStroke(true);
+        writer.writeElement(rect);
+    }
+
+
+    private static void writeText(String string,
+                                  Matrix2D matrix2D,
+                                  ElementBuilder builder,
+                                  Font helvetica,
+                                  double fontSize,
+                                  ElementWriter writer,
+                                  Color color) throws PDFNetException {
+
+        Element text = builder.createTextBegin(helvetica, fontSize);
+        text.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
+        float[] colorComponents = color.getColorComponents(null);
+        try (ColorPt colorpt = new ColorPt(colorComponents[0], colorComponents[1], colorComponents[2])) {
+            text.getGState().setFillColor(colorpt);
+        }
+        text.setTextMatrix(matrix2D);
+        text.getGState().setTextRenderMode(GState.e_fill_text);
+        writer.writeElement(text);
+
+        text = builder.createTextRun(string);
+        writer.writeElement(text);
+        text = builder.createTextEnd();
+        writer.writeElement(text);
+
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -4,18 +4,13 @@ import java.io.File;
 import java.nio.file.Path;
 import java.util.Map;
 
-import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
-import org.mockito.MockitoAnnotations;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
@@ -26,10 +21,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.visualizati
 import com.knecon.fforesight.service.layoutparser.server.PDFNetInitializer;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;
-import com.knecon.fforesight.tenantcommons.TenantsClient;
-import com.pdftron.pdf.PDFNet;
 
-import jakarta.annotation.PostConstruct;
 import lombok.SneakyThrows;
 
 public class ViewerDocumentTest extends BuildDocumentTest {
@@ -59,7 +51,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
 
         long start = System.currentTimeMillis();
         Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
-        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
+        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }
 
@@ -87,7 +79,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
         Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE_OLD, classificationDocument);
 
-        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
+        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/model/TextPositionSequenceTest.java

@@ -1,60 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server.model;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.io.ByteArrayInputStream;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.pdfbox.util.Matrix;
-import org.junit.jupiter.api.Test;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.storage.commons.properties.StorageProperties;
-import com.iqser.red.storage.commons.service.ObjectSerializer;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-
-import lombok.SneakyThrows;
-
-public class TextPositionSequenceTest {
-
-    private static final String TEXT_POSITION_SEQUENCE_AS_JSON = "{\n" //
-            + "  \"page\": 1,\n" //
-            + "  \"textPositions\": [],\n" //
-            + "  \"dir\": 180.0,\n" //
-            + "  \"rotation\": 0,\n" //
-            + "  \"pageHeight\": 800,\n" //
-            + "  \"pageWidth\": 600\n"  //
-            + "}";
-
-    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper());
-
-
-    @Test
-    @SneakyThrows
-    public void testDeserializationWithJackson() {
-
-        TextPositionSequence textPositionSequence = objectSerializer.deserialize(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
-                TextPositionSequence.class);
-
-        assertPropertiesAfterJsonDeserialization(textPositionSequence);
-    }
-
-
-    private void assertPropertiesAfterJsonDeserialization(TextPositionSequence textPositionSequence) {
-
-        assertThat(textPositionSequence.getPage()).isEqualTo(1);
-        assertThat(textPositionSequence.getTextPositions()).hasSize(0);
-        assertThat(textPositionSequence.getDir()).isEqualTo(TextDirection.HALF_CIRCLE);
-        assertThat(textPositionSequence.getRotation()).isEqualTo(0);
-        assertThat(textPositionSequence.getPageHeight()).isEqualTo(800f);
-        assertThat(textPositionSequence.getPageWidth()).isEqualTo(600f);
-    }
-
-
-    private Matrix createIdentityMatrix() {
-
-        return new Matrix();
-    }
-
-}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/PageContentExtractorTest.java

@@ -3,12 +3,10 @@ package com.knecon.fforesight.service.layoutparser.server.services;
 import java.nio.file.Path;
 import java.util.List;
 
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
 
@@ -29,7 +27,7 @@ class PageContentExtractorTest {
                 textPositionPerPage.stream()
                         .map(t -> t.getSortedTextPositionSequences()
                                 .stream()
-                                .map(TextPositionSequence::getBBoxInitialUserSpace)
+                                .map(TextPositionSequence::getBBoxPdf)
                                 .map(List::of)
                                 .toList())
                         .toList(), tmpFileName);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -10,7 +10,9 @@ import java.util.Optional;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -53,6 +55,11 @@ public abstract class AbstractTest {
     @MockBean
     private RabbitTemplate rabbitTemplate;
 
+    @MockBean
+    private RabbitAdmin rabbitAdmin;
+
+    @MockBean
+    private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
     protected final static String ORIGIN_FILE_ID = "origin";
 
     protected final static String VISUAL_LAYOUT_FILE = "visual";
@@ -141,16 +148,25 @@ public abstract class AbstractTest {
     @SneakyThrows
     protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile, String visualLayoutParsingResponseFile) {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
         ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
         ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
         ClassPathResource visualLayoutParsingResponseResource = new ClassPathResource(visualLayoutParsingResponseFile);
+        if (file.startsWith("/")) {
+            try (InputStream fileInputStream = new FileInputStream(file)) {
+                return prepareStorage(Path.of(file).getFileName().toString(),
+                                      fileInputStream,
+                                      cvServiceResponseFileResource.getInputStream(),
+                                      imageInfoFileResource.getInputStream(),
+                                      visualLayoutParsingResponseResource.getInputStream());
+            }
+        } else {
+            return prepareStorage(Path.of(file).getFileName().toString(),
+                                  new ClassPathResource(file).getInputStream(),
+                                  cvServiceResponseFileResource.getInputStream(),
+                                  imageInfoFileResource.getInputStream(),
+                                  visualLayoutParsingResponseResource.getInputStream());
+        }
 
-        return prepareStorage(Path.of(file).getFileName().toString(),
-                              pdfFileResource.getInputStream(),
-                              cvServiceResponseFileResource.getInputStream(),
-                              imageInfoFileResource.getInputStream(),
-                              visualLayoutParsingResponseResource.getInputStream());
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -1,6 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;
 
+import java.awt.geom.Rectangle2D;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.nio.file.Path;
 import java.util.Map;
 
@@ -10,11 +12,27 @@ import org.springframework.core.io.ClassPathResource;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.viewerdoc.model.Standard14EmbeddableFont;
+import com.pdftron.common.Matrix2D;
+import com.pdftron.pdf.ColorPt;
+import com.pdftron.pdf.ColorSpace;
+import com.pdftron.pdf.Element;
+import com.pdftron.pdf.ElementBuilder;
+import com.pdftron.pdf.ElementWriter;
+import com.pdftron.pdf.Font;
+import com.pdftron.pdf.GState;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.sdf.SDFDoc;
 
 import lombok.SneakyThrows;
 
@@ -27,7 +45,12 @@ public abstract class BuildDocumentTest extends AbstractTest {
     @SneakyThrows
     protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {
 
-        File fileResource = new ClassPathResource(filename).getFile();
+        File fileResource;
+        if (filename.startsWith("/")) {
+            fileResource = new File(filename);
+        } else {
+            fileResource = new ClassPathResource(filename).getFile();
+        }
         prepareStorage(filename);
         return layoutParsingPipeline.parseLayout(layoutParsingType,
                                                  fileResource,
@@ -48,14 +71,14 @@ public abstract class BuildDocumentTest extends AbstractTest {
     @SneakyThrows
     protected Document buildGraph(String filename, LayoutParsingType layoutParsingType) {
 
-
         if (!filename.startsWith("files") && filename.startsWith("/")) {
-            LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(Path.of(filename).getFileName().toString(), LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, true);
+            LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(Path.of(filename).getFileName().toString(), layoutParsingType, true);
             prepareStorage(layoutParsingRequest, new File(filename));
             return DocumentGraphFactory.buildDocumentGraph(layoutParsingType,
                                                            layoutParsingPipeline.parseLayout(layoutParsingType,
                                                                                              new File(filename),
-                                                                                             layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get()),
+                                                                                             layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId()
+                                                                                                                                               .get()),
                                                                                              new TableServiceResponse(),
                                                                                              new VisualLayoutParsingResponse(),
                                                                                              layoutParsingRequest.identifier()));
@@ -65,7 +88,8 @@ public abstract class BuildDocumentTest extends AbstractTest {
             } else {
                 prepareStorage(filename);
             }
-            return DocumentGraphFactory.buildDocumentGraph(layoutParsingType, parseLayout(filename, layoutParsingType));
+            var classificationDocument = parseLayout(filename, layoutParsingType);
+            return DocumentGraphFactory.buildDocumentGraph(layoutParsingType, classificationDocument);
         }
 
     }

layoutparser-service/layoutparser-service-server/src/test/resources/application.yml

@@ -39,3 +39,5 @@ management:
     prometheus.enabled: ${monitoring.enabled:false}
     health.enabled: true
   endpoints.web.exposure.include: prometheus, health
+
+POD_NAME: layoutparser-service