RED-7141: Fixed missing spaces in ExpansionTest.pdf

- added double column detection logic
- removing non y-intersecting left/right zones and adding them to middle for manual sorting

.gitignore

@@ -42,5 +42,3 @@ gradlew.bat
 gradlew
 gradle.properties
 gradle/
-.DS_Store
-.DS_Store/

.gitlab-ci.yml

@@ -1,7 +1,3 @@
-variables:
-  #  SONAR_PROJECT_KEY: 'fforesight_layout-parser_AYd5quv2mRkBOCG22hvF'
-  GIT_SUBMODULE_STRATEGY: recursive
-  GIT_SUBMODULE_FORCE_HTTPS: 'true'
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'

.gitmodules

@@ -1,8 +0,0 @@
-[submodule "layoutparser-service/layoutparser-service-server/src/test/resources/files/basf"]
-	path = layoutparser-service/layoutparser-service-server/src/test/resources/files/basf
-	url = https://gitlab.knecon.com/fforesight/documents/basf.git
-	update = merge
-[submodule "layoutparser-service/layoutparser-service-server/src/test/resources/files/syngenta"]
-	path = layoutparser-service/layoutparser-service-server/src/test/resources/files/syngenta
-	url = https://gitlab.knecon.com/fforesight/documents/syngenta.git
-	update = merge

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -3,5 +3,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 public class LayoutParsingQueueNames {
 
     public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
+    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_dead_letter_queue";
     public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java

@@ -1,10 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public enum LayoutParsingType {
-    REDACT_MANAGER,
     REDACT_MANAGER_OLD,
-    REDACT_MANAGER_PARAGRAPH_DEBUG,
+    TAAS,
     DOCUMINE,
-    CLARIFYND,
-    CLARIFYND_PARAGRAPH_DEBUG
+
+    DOCSTRUM,
+    REDACT_MANAGER
 }

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -24,6 +24,4 @@ dependencies {
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
-    implementation("org.jgrapht:jgrapht-core:1.5.2")
-    implementation("org.tinspin:tinspin-indexes:2.1.3")
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -27,7 +27,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
@@ -48,11 +47,11 @@ import com.knecon.fforesight.service.layoutparser.processor.services.TableExtrac
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClarifyndClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.services.graphics.GraphicExtractorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
@@ -79,20 +78,20 @@ public class LayoutParsingPipeline {
     CvTableParsingAdapter cvTableParsingAdapter;
     LayoutParsingStorageService layoutParsingStorageService;
     SectionsBuilderService sectionsBuilderService;
+    TaasClassificationService taasClassificationService;
     RedactManagerClassificationService redactManagerClassificationService;
     DocuMineClassificationService docuMineClassificationService;
     SimplifiedSectionTextService simplifiedSectionTextService;
     BodyTextFrameService bodyTextFrameService;
     RulingCleaningService rulingCleaningService;
     TableExtractionService tableExtractionService;
+    TaasBlockificationService taasBlockificationService;
     DocuMineBlockificationService docuMineBlockificationService;
     RedactManagerBlockificationService redactManagerBlockificationService;
     DocstrumBlockificationService docstrumBlockificationService;
     LayoutGridService layoutGridService;
     ObservationRegistry observationRegistry;
     VisualLayoutParsingAdapter visualLayoutParsingAdapter;
-    ClarifyndClassificationService clarifyndClassificationService;
-    GraphicExtractorService graphicExtractorService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -123,11 +122,11 @@ public class LayoutParsingPipeline {
                 imageServiceResponse,
                 tableServiceResponse,
                 visualLayoutParsingResponse,
-                layoutParsingRequest.identifier());
+                layoutParsingRequest.identifier().toString());
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
-        Document documentGraph = observeBuildDocumentGraph(layoutParsingRequest.layoutParsingType(), classificationDocument);
+        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
 
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
 
@@ -139,7 +138,7 @@ public class LayoutParsingPipeline {
         layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
         layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
 
-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.CLARIFYND)) {
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
             log.info("Building research document data for {}", layoutParsingRequest.identifier());
             var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
             layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
@@ -179,13 +178,13 @@ public class LayoutParsingPipeline {
     }
 
 
-    private Document observeBuildDocumentGraph(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument) {
+    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
 
         AtomicReference<Document> documentReference = new AtomicReference<>();
 
         Observation.createNotStarted("LayoutParsingPipeline", observationRegistry)
                 .contextualName("build-document-graph")
-                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(layoutParsingType, classificationDocument)));
+                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument)));
 
         return documentReference.get();
     }
@@ -212,7 +211,7 @@ public class LayoutParsingPipeline {
                                               ImageServiceResponse imageServiceResponse,
                                               TableServiceResponse tableServiceResponse,
                                               VisualLayoutParsingResponse visualLayoutParsingResponse,
-                                              Map<String, String> identifier) {
+                                              String identifier) {
 
         PDDocument originDocument = openDocument(originFile);
         addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
@@ -259,26 +258,14 @@ public class LayoutParsingPipeline {
 
             List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
 
-            var graphics = graphicExtractorService.extractPathElementGraphics(originDocument,
-                    pdPage,
-                    pageNumber,
-                    cleanRulings,
-                    stripper.getTextPositionSequences(),
-                    emptyTableCells,
-                    false);
-
-            pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>())
-                    .addAll(graphics.stream()
-                            .map(g -> new ClassifiedImage(new Rectangle2D.Double(g.x1, g.y1, g.width(), g.height()), ImageType.GRAPHIC, false, stripper.getPageNumber()))
-                            .toList());
-
             ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER_OLD -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells);
+                case REDACT_MANAGER_OLD ->
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
                 case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, true);
-                case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, false);
+                case DOCSTRUM -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, false);
+                case REDACT_MANAGER -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, true);
             };
-
             classificationPage.setCleanRulings(cleanRulings);
             classificationPage.setRotation(rotation);
             classificationPage.setLandscape(isLandscape);
@@ -305,10 +292,8 @@ public class LayoutParsingPipeline {
 
             tableExtractionService.extractTables(emptyTableCells, classificationPage);
 
-            if (layoutParsingType == LayoutParsingType.REDACT_MANAGER) {
+            if (layoutParsingType == LayoutParsingType.DOCSTRUM || layoutParsingType == LayoutParsingType.REDACT_MANAGER) {
                 docstrumBlockificationService.combineBlocks(classificationPage);
-            } else if (layoutParsingType == LayoutParsingType.CLARIFYND) {
-                docstrumBlockificationService.mergeIntersectingBlocks(classificationPage.getTextBlocks(), 0, 6.5f);
             }
 
             buildPageStatistics(classificationPage);
@@ -323,22 +308,30 @@ public class LayoutParsingPipeline {
         bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
         log.info("Classify TextBlocks for {}", identifier);
         switch (layoutParsingType) {
-            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG ->
-                    redactManagerClassificationService.classifyDocument(classificationDocument);
+            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
             case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER_OLD -> redactManagerClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
+            case DOCSTRUM -> redactManagerClassificationService.classifyDocument(classificationDocument);
         }
 
         log.info("Building Sections for {}", identifier);
 
-        switch (layoutParsingType) {
-            case CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_PARAGRAPH_DEBUG -> sectionsBuilderService.buildParagraphDebugSections(classificationDocument);
-            default -> {
-                sectionsBuilderService.buildSections(classificationDocument);
-                sectionsBuilderService.addImagesToSections(classificationDocument);
-            }
-        }
-
+//        if (layoutParsingType == DOCSTRUM || layoutParsingType == DOCSTRUM_XY) {
+//            // Currently for debugging return paragraphs as sections, because there is a merging logic in sectionBuilder
+//            List<ClassificationSection> sections = new ArrayList<>();
+//            for (var page : classificationPages) {
+//                page.getTextBlocks().forEach(block -> {
+//                    block.setPage(page.getPageNumber());
+//                    var section = sectionsBuilderService.buildTextBlock(List.of(block), "a");
+//                    sections.add(section);
+//                });
+//            }
+//            classificationDocument.setSections(sections);
+//        } else {
+        sectionsBuilderService.buildSections(classificationDocument);
+        sectionsBuilderService.addImagesToSections(classificationDocument);
+//        }
         return classificationDocument;
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/UnionFind.java

@@ -1,31 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
-
-import java.util.Collection;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Set;
-
-public class UnionFind<T> extends org.jgrapht.alg.util.UnionFind<T> {
-
-    public UnionFind(Set<T> elements) {
-
-        super(elements);
-    }
-
-
-    public Collection<Set<T>> getGroups() {
-
-        Map<T, Set<T>> setRep = new LinkedHashMap<>();
-        for (T t : getParentMap().keySet()) {
-            T representative = find(t);
-            if (!setRep.containsKey(representative)) {
-                setRep.put(representative, new LinkedHashSet<>());
-            }
-            setRep.get(representative).add(t);
-        }
-
-        return setRep.values();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/NearestNeighbourService.java

@@ -1,36 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
-
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-import org.tinspin.index.Index;
-import org.tinspin.index.kdtree.KDTree;
-
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;
-
-@Service
-public class NearestNeighbourService {
-
-    private static final int NUMBER_OF_NEIGHBOURS = 8;
-
-
-    public void findNearestNeighbors(List<Character> characters) {
-        KDTree<Character> kdTree = KDTree.create(2);
-        characters.forEach(c -> kdTree.insert(new double[]{c.getX(), c.getY()}, c));
-
-        for(Character c : characters) {
-            Index.PointIteratorKnn<Character> iterator = kdTree.queryKnn(new double[]{c.getX(), c.getY()}, NUMBER_OF_NEIGHBOURS + 1);
-            // skip the first as this is identity
-            if(iterator.hasNext()) {
-                iterator.next();
-            }
-            while(iterator.hasNext()) {
-                c.getNeighbors().add(new Neighbor(iterator.next().value(), c));
-            }
-        }
-
-    }
-
-}
-

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/utils/FastAtan2.java

@@ -1,76 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.utils;
-
-public class FastAtan2 {
-
-    static final private int Size_Ac = 1000;
-    static final private int Size_Ar = Size_Ac + 1;
-    static final private double Pi = (float) Math.PI;
-    static final private double Pi_H = Pi / 2;
-
-    static final private double[] Atan2 = new double[Size_Ar];
-    static final private double[] Atan2_PM = new double[Size_Ar];
-    static final private double[] Atan2_MP = new double[Size_Ar];
-    static final private double[] Atan2_MM = new double[Size_Ar];
-
-    static final private double[] Atan2_R = new double[Size_Ar];
-    static final private double[] Atan2_RPM = new double[Size_Ar];
-    static final private double[] Atan2_RMP = new double[Size_Ar];
-    static final private double[] Atan2_RMM = new double[Size_Ar];
-
-    static {
-        for (int i = 0; i <= Size_Ac; i++) {
-            double d = (double) i / Size_Ac;
-            double x = 1;
-            double y = x * d;
-            double v = Math.atan2(y, x);
-            Atan2[i] = v;
-            Atan2_PM[i] = Pi - v;
-            Atan2_MP[i] = -v;
-            Atan2_MM[i] = -Pi + v;
-
-            Atan2_R[i] = Pi_H - v;
-            Atan2_RPM[i] = Pi_H + v;
-            Atan2_RMP[i] = -Pi_H + v;
-            Atan2_RMM[i] = -Pi_H - v;
-
-        }
-    }
-
-    @SuppressWarnings("ParameterAssignment")
-    static public double fastAtan2(double y, double x) {
-
-        if (y < 0) {
-            if (x < 0) {
-                //(y < x) because == (-y > -x)
-                if (y < x) {
-                    return Atan2_RMM[(int) (x / y * Size_Ac)];
-                } else {
-                    return Atan2_MM[(int) (y / x * Size_Ac)];
-                }
-            } else {
-                y = -y;
-                if (y > x) {
-                    return Atan2_RMP[(int) (x / y * Size_Ac)];
-                } else {
-                    return Atan2_MP[(int) (y / x * Size_Ac)];
-                }
-            }
-        } else {
-            if (x < 0) {
-                x = -x;
-                if (y > x) {
-                    return Atan2_RPM[(int) (x / y * Size_Ac)];
-                } else {
-                    return Atan2_PM[(int) (y / x * Size_Ac)];
-                }
-            } else {
-                if (y > x) {
-                    return Atan2_R[(int) (x / y * Size_Ac)];
-                } else {
-                    return Atan2[(int) (y / x * Size_Ac)];
-                }
-            }
-        }
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java

@@ -64,24 +64,9 @@ public class Document implements GenericSemanticNode {
     }
 
 
-    public List<Header> getHeaders() {
-
-        return streamChildrenOfType(NodeType.HEADER).map(node -> (Header) node)
-                .collect(Collectors.toList());
-    }
-
-
-    public List<Footer> getFooters() {
-
-        return streamChildrenOfType(NodeType.FOOTER).map(node -> (Footer) node)
-                .collect(Collectors.toList());
-    }
-
-
     public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
 
-        return streamAllNodes().filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getTextBlock);
+        return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getTextBlock);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java

@@ -13,6 +13,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.No
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -39,8 +40,6 @@ public class Image implements GenericSemanticNode {
     boolean transparent;
     Rectangle2D position;
 
-    TextBlock leafTextBlock;
-
     boolean redaction;
     boolean ignored;
     @Builder.Default
@@ -71,7 +70,9 @@ public class Image implements GenericSemanticNode {
     @Override
     public TextBlock getTextBlock() {
 
-        return leafTextBlock;
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -97,11 +98,4 @@ public class Image implements GenericSemanticNode {
         return bBoxPerPage;
     }
 
-
-    @Override
-    public boolean isLeaf() {
-
-        return true;
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java

@@ -9,8 +9,7 @@ public enum ImageType {
 
     SIGNATURE_VISUAL,
     OTHER,
-    OCR,
-    GRAPHIC;
+    OCR;
 
 
     public static ImageType fromString(String imageType) {
@@ -20,7 +19,6 @@ public enum ImageType {
             case "formula" -> ImageType.FORMULA;
             case "signature" -> ImageType.SIGNATURE;
             case "ocr" -> ImageType.OCR;
-            case "graphic" -> ImageType.GRAPHIC;
             default -> ImageType.OTHER;
         };
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import org.apache.pdfbox.text.TextPosition;
+import org.springframework.beans.BeanUtils;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
@@ -49,15 +50,11 @@ public class RedTextPosition {
     public static RedTextPosition fromTextPosition(TextPosition textPosition) {
 
         var pos = new RedTextPosition();
-        pos.setRotation(textPosition.getRotation());
-        pos.setPageHeight(textPosition.getPageHeight());
-        pos.setPageWidth(textPosition.getPageWidth());
-        pos.setUnicode(textPosition.getUnicode());
-        pos.setDir(textPosition.getDir());
-        pos.setWidthOfSpace(textPosition.getWidthOfSpace());
-        pos.setFontSizeInPt(textPosition.getFontSizeInPt());
+        BeanUtils.copyProperties(textPosition, pos);
         pos.setFontName(textPosition.getFont().getName());
 
+        pos.setFontSizeInPt(textPosition.getFontSizeInPt());
+
         var position = new float[4];
 
         position[0] = textPosition.getXDirAdj();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -15,7 +15,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -25,18 +24,12 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TextPositionSequence implements CharSequence {
 
     public static final int HEIGHT_PADDING = 2;
-
-    @EqualsAndHashCode.Include
     private int page;
-
-    @EqualsAndHashCode.Include
     private List<RedTextPosition> textPositions = new ArrayList<>();
 
-    @EqualsAndHashCode.Include
     private TextDirection dir;
     private int rotation;
     private float pageHeight;
@@ -208,12 +201,6 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public float getTextHeightNoPadding() {
-
-        return textPositions.get(0).getHeightDir();
-    }
-
-
     public float getTextHeight() {
 
         return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/ImageServiceResponseAdapter.java

@@ -9,10 +9,10 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 
 import lombok.RequiredArgsConstructor;
 
@@ -20,7 +20,8 @@ import lombok.RequiredArgsConstructor;
 @RequiredArgsConstructor
 public class ImageServiceResponseAdapter {
 
-    public Map<Integer, List<ClassifiedImage>> buildClassifiedImagesPerPage(ImageServiceResponse imageServiceResponse) {
+
+    public Map<Integer, List<ClassifiedImage>> buildClassifiedImagesPerPage(ImageServiceResponse imageServiceResponse ) {
 
         Map<Integer, List<ClassifiedImage>> images = new HashMap<>();
         imageServiceResponse.getData().forEach(imageMetadata -> {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/BodyTextFrameService.java

@@ -25,7 +25,6 @@ public class BodyTextFrameService {
     private static final float RULING_HEIGHT_THRESHOLD = 0.15f; // multiplied with page height. Header/Footer Rulings must be within that border of the page.
     private static final float RULING_WIDTH_THRESHOLD = 0.75f; // multiplied with page width. Header/Footer Rulings must be at least that wide.
 
-
     public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {
 
         Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
@@ -133,7 +132,12 @@ public class BodyTextFrameService {
                                                boolean landscape,
                                                LayoutParsingType layoutParsingType) {
 
-        float approximateHeaderLineCount = 2.9f;
+        float approximateHeaderLineCount;
+        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
+            approximateHeaderLineCount = 3.3f;
+        } else {
+            approximateHeaderLineCount = 2.9f;
+        }
 
         BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();
 
@@ -151,9 +155,8 @@ public class BodyTextFrameService {
                         continue;
                     }
 
-                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || MarkedContentUtils.intersects(textBlock,
-                            page.getMarkedContentBboxPerType(),
-                            MarkedContentUtils.FOOTER)) {
+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
                         continue;
                     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionsBuilderService.java

@@ -7,7 +7,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import org.apache.logging.log4j.util.Strings;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -44,6 +43,7 @@ public class SectionsBuilderService {
         for (ClassificationPage page : document.getPages()) {
             List<TextPageBlock> header = new ArrayList<>();
             List<TextPageBlock> footer = new ArrayList<>();
+            List<TextPageBlock> unclassifiedText = new ArrayList<>();
             for (AbstractPageBlock current : page.getTextBlocks()) {
 
                 if (current.getClassification() == null) {
@@ -62,6 +62,11 @@ public class SectionsBuilderService {
                     continue;
                 }
 
+                if (current.getClassification().equals(PageBlockType.OTHER)) {
+                    unclassifiedText.add((TextPageBlock) current);
+                    continue;
+                }
+
                 if (prev != null && current.getClassification().isHeadline() && !prev.getClassification().isHeadline() || !document.isHeadlines()) {
                     ClassificationSection chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
@@ -89,6 +94,9 @@ public class SectionsBuilderService {
             if (!footer.isEmpty()) {
                 footers.add(new ClassificationFooter(footer));
             }
+            if (!unclassifiedText.isEmpty()) {
+                unclassifiedTexts.add(new UnclassifiedText(unclassifiedText));
+            }
         }
 
         ClassificationSection chunkBlock = buildTextBlock(chunkWords, lastHeadline);
@@ -102,20 +110,6 @@ public class SectionsBuilderService {
     }
 
 
-    public void buildParagraphDebugSections(ClassificationDocument document) {
-
-        List<ClassificationSection> sections = new ArrayList<>();
-        for (var page : document.getPages()) {
-            page.getTextBlocks().forEach(block -> {
-                block.setPage(page.getPageNumber());
-                var section = buildTextBlock(List.of(block), Strings.EMPTY);
-                sections.add(section);
-            });
-        }
-        document.setSections(sections);
-    }
-
-
     public void addImagesToSections(ClassificationDocument document) {
 
         Map<Integer, List<ClassificationSection>> sectionMap = new HashMap<>();
@@ -206,7 +200,6 @@ public class SectionsBuilderService {
                             .getX() <= xMax && image.getPosition().getY() >= yMin && image.getPosition().getY() <= yMax) {
                         section.getImages().add(image);
                         image.setAppendedToSection(true);
-                        break;
                     }
                 }
                 if (!image.isAppendedToSection()) {
@@ -247,7 +240,7 @@ public class SectionsBuilderService {
     }
 
 
-    private ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
+    public ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
 
         ClassificationSection section = new ClassificationSection();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -1,51 +1,26 @@
 package com.knecon.fforesight.service.layoutparser.processor.services;
 
 import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-
-import lombok.extern.slf4j.Slf4j;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 
 @Service
-@Slf4j
 public class SimplifiedSectionTextService {
 
     public SimplifiedText toSimplifiedText(Document document) {
 
-        List<SimplifiedSectionText> simplifiedMainSectionsList = document.getMainSections()
-                .stream()
-                .map(this::toSimplifiedSectionText)
-                .toList();
-        List<SimplifiedSectionText> simplifiedHeadersList = document.getHeaders()
-                .stream()
-                .map(this::toSimplifiedSectionText)
-                .toList();
-        List<SimplifiedSectionText> simplifiedFootersList = document.getFooters()
-                .stream()
-                .map(this::toSimplifiedSectionText)
-                .toList();
-        List<SimplifiedSectionText> simplifiedText = Stream.of(simplifiedMainSectionsList, simplifiedHeadersList, simplifiedFootersList)
-                .flatMap(List::stream)
-                .collect(Collectors.toList());
-        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedText).build();
+        List<SimplifiedSectionText> simplifiedSectionTexts = document.getMainSections().stream().map(this::toSimplifiedSectionText).toList();
+        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build();
     }
 
 
-    private SimplifiedSectionText toSimplifiedSectionText(SemanticNode section) {
+    private SimplifiedSectionText toSimplifiedSectionText(Section section) {
 
-        return SimplifiedSectionText.builder()
-                .sectionNumber(section.getTreeId()
-                                       .get(0))
-                .text(section.getTextBlock().getSearchText())
-                .build();
+        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
     }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -2,28 +2,29 @@ package com.knecon.fforesight.service.layoutparser.processor.services.blockifica
 
 import static java.util.stream.Collectors.toSet;
 
+import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.ListIterator;
+import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.DocstrumSegmentationService;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionSequenceComparator;
 
 import lombok.RequiredArgsConstructor;
 
@@ -39,22 +40,19 @@ public class DocstrumBlockificationService {
 
     public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Cell> cells, boolean xyOrder) {
 
-        CleanRulings usedRulings = RectangleTransformations.extractRulings(cells);
+        // Underlined or strikethrough are also in rulings but we dont want to split blocks with them so we use cells.
+        List<Ruling> usedHorizonalRulings = new ArrayList<>();
+        List<Ruling> usedVerticalRulings = new ArrayList<>();
 
-        var zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder);
-        var pageBlocks = toAbstractPageBlocks(zones, usedRulings.getHorizontal(), usedRulings.getVertical(), xyOrder);
-
-        var classificationPage = new ClassificationPage(pageBlocks);
-
-        mergeIntersectingBlocks(classificationPage.getTextBlocks(), 0, 0);
-
-        return classificationPage;
-    }
-
-
-    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones, List<Ruling> horizontalRulings, List<Ruling> verticalRulings, boolean xyOrder) {
+        cells.forEach(cell -> {
+            usedHorizonalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y), new Point2D.Float(cell.x + cell.width, cell.y)));
+            usedHorizonalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y + cell.height), new Point2D.Float(cell.x + cell.width, cell.y + cell.height)));
+            usedVerticalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y), new Point2D.Float(cell.x, cell.y + cell.height)));
+            usedVerticalRulings.add(new Ruling(new Point2D.Float(cell.x + cell.width, cell.y), new Point2D.Float(cell.x + cell.width, cell.y + cell.height)));
+        });
 
         List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
+        var zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder);
         zones.forEach(zone -> {
 
             List<TextPositionSequence> textPositionSequences = new ArrayList<>();
@@ -64,62 +62,83 @@ public class DocstrumBlockificationService {
                 });
             });
 
-            abstractPageBlocks.addAll(splitZonesAtRulings(textPositionSequences, horizontalRulings, verticalRulings));
+            abstractPageBlocks.addAll(splitZonesAtRulings(textPositionSequences, usedHorizonalRulings, usedVerticalRulings));
+//            abstractPageBlocks.add(buildTextBlock(textPositionSequences, 0));
         });
 
-        if (xyOrder) {
-            abstractPageBlocks.sort(Comparator.comparing(AbstractPageBlock::getMinY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                    .thenComparing(AbstractPageBlock::getMinX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-            abstractPageBlocks.sort(new Comparator<AbstractPageBlock>() {
-                @Override
-                public int compare(AbstractPageBlock o1, AbstractPageBlock o2) {
-
-                    return Math.abs(o1.getMinY() - o2.getMinY()) < 5 && o1.getMinX() < o2.getMinX() == true ? -1 : 0;
-                }
-            });
-        }
-
-        return abstractPageBlocks;
+        return new ClassificationPage(abstractPageBlocks);
     }
 
 
     public void combineBlocks(ClassificationPage page) {
 
+        mergeZones(page.getTextBlocks());
+
         TextPageBlock previous = new TextPageBlock();
         ListIterator<AbstractPageBlock> itty = page.getTextBlocks().listIterator();
         while (itty.hasNext()) {
-
             AbstractPageBlock block = itty.next();
             if (block instanceof TablePageBlock) {
-                previous = new TextPageBlock();
                 continue;
             }
             TextPageBlock current = (TextPageBlock) block;
 
             if (previous != null && !previous.getSequences().isEmpty()) {
 
-                if (current.getDir() != previous.getDir()) {
-                    previous = current;
+                if (current.getDir() == previous.getDir() //
+                        && previous.getNumberOfLines() >= 2 && current.getNumberOfLines() >= 2 //
+                        && previous.intersectsY(current) //
+                        && !hasBetween(current, previous, page.getTextBlocks()) //
+                        && numberOfYIntersections(current, previous, page.getTextBlocks()) == 0) {
+
+                    previous.getSequences().addAll(current.getSequences());
+                    previous = buildTextBlock(previous.getSequences(), 0);
+                    previous.setToDuplicate(true);
+                    itty.remove();
+                    itty.previous();
+                    itty.set(previous);
+                    itty.next();
                     continue;
                 }
 
-                if (areTheOnlyTwoBlocksOnHeightsWithBothMoreThanTwoLines(previous, current, page)) {
-                    previous = combineBlocksAndResetIterator(previous, current, itty, true);
+                if (current.getDir() == previous.getDir() && (previous.almostIntersects(current, 0, 0))) {
+
+                    previous.getSequences().addAll(current.getSequences());
+                    boolean toDuplicate = previous.isToDuplicate();
+                    previous = buildTextBlock(previous.getSequences(), 0);
+                    previous.setToDuplicate(toDuplicate);
+                    itty.remove();
+                    itty.previous();
+                    itty.set(previous);
+                    itty.next();
                     continue;
                 }
 
-                if (previous.almostIntersects(current, 0, 0)) {
-                    previous = combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
+                if (current.getDir() == previous.getDir() //
+                        && (Math.abs(previous.getMaxY() - current.getMaxY()) < THRESHOLD || Math.abs(previous.getMinY() - current.getMinY()) < THRESHOLD) //
+                        && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
+                        && !hasBetween(current, previous, page.getTextBlocks()) && numberOfYIntersections(current, previous, page.getTextBlocks()) <= 4) {
+
+                    previous.getSequences().addAll(current.getSequences());
+                    previous = buildTextBlock(previous.getSequences(), 0);
+                    itty.remove();
+                    itty.previous();
+                    itty.set(previous);
+                    itty.next();
                     continue;
                 }
 
-                if (isSameTopOrBottomWithPreviousHavingMaxTwoLinesAndCurrentThanOneAndMax4OtherBlocksOnHeight(previous, current, page)) {
-                    previous = combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
-                    continue;
-                }
-
-                if (isOnlyIntersectingYAndOnelineOrPrevoiusTwoLines(previous, current, page)) {
-                    previous = combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
+                if (current.getDir() == previous.getDir() //
+                        && current.intersectsY(previous) //
+                        && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
+                        && !hasBetween(current, previous, page.getTextBlocks()) //
+                        && numberOfYIntersections(current, previous, page.getTextBlocks()) <= 0) {
+                    previous.getSequences().addAll(current.getSequences());
+                    previous = buildTextBlock(previous.getSequences(), 0);
+                    itty.remove();
+                    itty.previous();
+                    itty.set(previous);
+                    itty.next();
                     continue;
                 }
 
@@ -127,46 +146,7 @@ public class DocstrumBlockificationService {
             previous = current;
         }
 
-        mergeIntersectingBlocks(page.getTextBlocks(), 0, 6.5f);
-    }
-
-
-    private boolean isOnlyIntersectingYAndOnelineOrPrevoiusTwoLines(TextPageBlock previous, TextPageBlock current, ClassificationPage page) {
-
-        return current.intersectsY(previous) //
-                && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
-                && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) <= 0;
-    }
-
-
-    private boolean isSameTopOrBottomWithPreviousHavingMaxTwoLinesAndCurrentThanOneAndMax4OtherBlocksOnHeight(TextPageBlock previous,
-                                                                                                              TextPageBlock current,
-                                                                                                              ClassificationPage page) {
-
-        return (Math.abs(previous.getMaxY() - current.getMaxY()) < THRESHOLD || Math.abs(previous.getMinY() - current.getMinY()) < THRESHOLD) //
-                && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() >= 2 && current.getNumberOfLines() == 1) //
-                && !hasBetween(current, previous, page.getTextBlocks()) && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) <= 4;
-    }
-
-
-    private boolean areTheOnlyTwoBlocksOnHeightsWithBothMoreThanTwoLines(TextPageBlock previous, TextPageBlock current, ClassificationPage page) {
-
-        return previous.getNumberOfLines() >= 2 && current.getNumberOfLines() >= 2 //
-                && previous.intersectsY(current) //
-                && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) == 0;
-    }
-
-
-    private TextPageBlock combineBlocksAndResetIterator(TextPageBlock previous, TextPageBlock current, ListIterator<AbstractPageBlock> itty, boolean toDuplicate) {
-
-        previous.getSequences().addAll(current.getSequences());
-        previous = buildTextBlock(previous.getSequences(), 0);
-        previous.setToDuplicate(toDuplicate);
-        itty.remove();
-        itty.previous();
-        itty.set(previous);
-        itty.next();
-        return previous;
+        mergeZones(page.getTextBlocks());
     }
 
 
@@ -187,7 +167,7 @@ public class DocstrumBlockificationService {
     }
 
 
-    private int numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(TextPageBlock block, TextPageBlock other, List<AbstractPageBlock> allBlocks) {
+    private int numberOfYIntersections(TextPageBlock block, TextPageBlock other, List<AbstractPageBlock> allBlocks) {
 
         double minY = Math.min(block.getMinY(), other.getMinY());
         double maxY = Math.min(block.getMaxY(), other.getMaxY());
@@ -208,52 +188,51 @@ public class DocstrumBlockificationService {
     }
 
 
-    public void mergeIntersectingBlocks(List<AbstractPageBlock> blocks, float xThreshold, float yThreshold) {
+    private void mergeZones(List<AbstractPageBlock> zones) {
 
-        ListIterator<AbstractPageBlock> itty = blocks.listIterator();
+        ListIterator<AbstractPageBlock> itty = zones.listIterator();
+        Set<AbstractPageBlock> toRemove = new HashSet<>();
         while (itty.hasNext()) {
             AbstractPageBlock block = itty.next();
-            if(block == null){
-                continue;
-            }
             if (block instanceof TablePageBlock) {
                 continue;
             }
 
             TextPageBlock current = (TextPageBlock) block;
 
-            for (int i = 0; i < blocks.size(); i++) {
+            if (current.isToDuplicate()) {
+                continue;
+            }
 
-                if(blocks.get(i) == null){
+            for (int i = 0; i < zones.size(); i++) {
+
+                if (toRemove.contains(zones.get(i))) {
                     continue;
                 }
-                if (blocks.get(i) == current) {
+                if (zones.get(i) == current) {
                     continue;
                 }
-                if (blocks.get(i) instanceof TablePageBlock) {
+                if (zones.get(i) instanceof TablePageBlock) {
                     continue;
                 }
 
-                TextPageBlock inner = (TextPageBlock) blocks.get(i);
+                TextPageBlock inner = (TextPageBlock) zones.get(i);
 
-                if (current.getDir() == inner.getDir() && current.almostIntersects(inner, yThreshold, xThreshold)) {
+                if (inner.isToDuplicate()) {
+                    continue;
+                }
+
+                if (current.getDir() == inner.getDir() && current.almostIntersects(inner, 0, 0)) {
 
-                    boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
                     current.getSequences().addAll(inner.getSequences());
+                    QuickSort.sort(current.getSequences(), new TextPositionSequenceComparator());
                     current = buildTextBlock(current.getSequences(), 0);
-
-                    current.setToDuplicate(toDuplicate);
-                    blocks.set(i, null);
+                    toRemove.add(inner);
                     itty.set(current);
                 }
             }
         }
-        var blocksIterator = blocks.iterator();
-        while(blocksIterator.hasNext()){
-            if(blocksIterator.next() == null){
-                blocksIterator.remove();
-            }
-        }
+        zones.removeAll(toRemove);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocuMineBlockificationService.java

@@ -61,8 +61,8 @@ public class DocuMineBlockificationService {
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
             boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 && (word.getFontStyle()
-                    .contains("bold") && !prev.getFontStyle().contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
+            boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle()
+                    .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
 
             Matcher matcher = pattern.matcher(chunkWords.stream().collect(Collectors.joining(" ")).toString());
             boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java

@@ -13,13 +13,10 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
 
 @SuppressWarnings("all")
@@ -34,12 +31,12 @@ public class RedactManagerBlockificationService {
      * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
      * Rulings (TablePageBlock lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
      *
-     * @param textPositions The words of a page.
+     * @param textPositions         The words of a page.
+     * @param horizontalRulingLines Horizontal table lines.
+     * @param verticalRulingLines   Vertical table lines.
      * @return Page object that contains the Textblock and text statistics.
      */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Cell> cells) {
-
-        CleanRulings usedRulings = RectangleTransformations.extractRulings(cells);
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
         int indexOnPage = 0;
         List<TextPositionSequence> chunkWords = new ArrayList<>();
@@ -57,7 +54,7 @@ public class RedactManagerBlockificationService {
             boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
             boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
-            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, usedRulings.getHorizontal(), usedRulings.getVertical());
+            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
 
             if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/TaasBlockificationService.java

@@ -0,0 +1,330 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.blockification;
+
+
+// TODO: figure out, why this fails the build
+// import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+@Service
+@SuppressWarnings("all")
+public class TaasBlockificationService {
+
+    private static final float THRESHOLD = 1f;
+    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f; // multiplied with text height
+    private static final float INTERSECTS_Y_THRESHOLD = 4;// 2 * HEIGHT_PADDING // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
+    private static final int X_GAP_SPLIT_CONSTANT = 50;
+    public static final int X_ALIGNMENT_THRESHOLD = 1;
+    public static final int NEGATIVE_X_GAP_THRESHOLD = -5;
+
+    private Pattern listIdentifier = Pattern.compile("^(?:(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)]))|\\uF0B7", Pattern.CASE_INSENSITIVE);
+
+
+    /**
+     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
+     * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
+     * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
+     *
+     * @param textPositions         The words of a page.
+     * @param horizontalRulingLines Horizontal table lines.
+     * @param verticalRulingLines   Vertical table lines.
+     * @return ClassificationPage object that contains the Textblock and text statistics.
+     */
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+
+        List<TextPageBlock> classificationTextBlocks = constructFineGranularTextPageBlocks(textPositions, horizontalRulingLines, verticalRulingLines);
+        classificationTextBlocks = mergeTextPageBlocksAligningX(classificationTextBlocks);
+        classificationTextBlocks = mergeIntersectingTextBlocksUntilConvergence(classificationTextBlocks);
+
+        return new ClassificationPage(new ArrayList<>(classificationTextBlocks.stream().map(classificationTextBlock -> (AbstractPageBlock) classificationTextBlock).toList()));
+    }
+
+
+    private List<TextPageBlock> mergeIntersectingTextBlocksUntilConvergence(List<TextPageBlock> classificationTextBlocks) {
+
+        int currentSize = classificationTextBlocks.size();
+        while (true) {
+            classificationTextBlocks = mergeTextPageBlocksAlmostIntersecting(classificationTextBlocks);
+            if (classificationTextBlocks.size() == currentSize) {
+                break;
+            }
+            currentSize = classificationTextBlocks.size();
+        }
+        return classificationTextBlocks;
+    }
+
+
+    private List<TextPageBlock> mergeTextPageBlocksAligningX(List<TextPageBlock> classificationTextBlocks) {
+
+        if (classificationTextBlocks.isEmpty()) {
+            return new ArrayList<>();
+        }
+        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
+        List<TextPageBlock> currentTextBlocksToMerge = new LinkedList<>();
+        textBlocksToMerge.add(currentTextBlocksToMerge);
+        TextPageBlock previousTextBlock = null;
+        Float lastLineGap = null;
+        for (TextPageBlock currentTextBlock : classificationTextBlocks) {
+            if (previousTextBlock == null) {
+                currentTextBlocksToMerge.add(currentTextBlock);
+                previousTextBlock = currentTextBlock;
+                continue;
+            }
+
+
+            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
+            boolean isListIdentifier = listIdentifierPattern.find();
+
+            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+
+            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
+//            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+
+            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
+            boolean alignsXLeft = Math.abs(currentTextBlock.getPdfMinX() - previousTextBlock.getPdfMinX()) < X_ALIGNMENT_THRESHOLD;
+//            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < yGap;
+            if (yGap && sameFont && !isListIdentifier) {
+                currentTextBlocksToMerge.add(currentTextBlock);
+
+            } else {
+                currentTextBlocksToMerge = new LinkedList<>();
+                currentTextBlocksToMerge.add(currentTextBlock);
+                textBlocksToMerge.add(currentTextBlocksToMerge);
+            }
+            previousTextBlock = currentTextBlock;
+        }
+        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
+    }
+
+
+    private List<TextPageBlock> mergeTextPageBlocksAlmostIntersecting(List<TextPageBlock> textPageBlocks) {
+
+        Set<TextPageBlock> alreadyMerged = new HashSet<>();
+        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
+        for (TextPageBlock textPageBlock : textPageBlocks) {
+            if (alreadyMerged.contains(textPageBlock)) {
+                continue;
+            }
+            alreadyMerged.add(textPageBlock);
+            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
+                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
+                    .toList());
+        }
+        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
+    }
+
+
+    private void assignOrientations(List<TextPageBlock> classificationTextBlocks) {
+
+        Iterator<TextPageBlock> itty = classificationTextBlocks.iterator();
+
+        TextPageBlock previousLeft = null;
+        TextPageBlock previousRight = null;
+        while (itty.hasNext()) {
+            TextPageBlock block = (TextPageBlock) itty.next();
+
+            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
+                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
+                    previousLeft.add(block);
+                    itty.remove();
+                    continue;
+                }
+            }
+
+            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
+                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
+                    previousRight.add(block);
+                    itty.remove();
+                    continue;
+                }
+            }
+
+            if (block.getOrientation().equals(Orientation.LEFT)) {
+                previousLeft = block;
+            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
+                previousRight = block;
+            }
+        }
+
+        itty = classificationTextBlocks.iterator();
+        TextPageBlock previous = null;
+        while (itty.hasNext()) {
+            TextPageBlock block = (TextPageBlock) itty.next();
+
+            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(
+                    block.getMaxY(),
+                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
+                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
+                previous.add(block);
+                itty.remove();
+                continue;
+            }
+
+            previous = block;
+        }
+    }
+
+
+    private List<TextPageBlock> constructFineGranularTextPageBlocks(List<TextPositionSequence> textPositions,
+                                                                    List<Ruling> horizontalRulingLines,
+                                                                    List<Ruling> verticalRulingLines) {
+
+        int indexOnPage = 0;
+        List<TextPositionSequence> wordClusterToCombine = new ArrayList<>();
+        List<TextPageBlock> classificationTextBlocks = new ArrayList<>();
+
+        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
+        TextPositionSequence prev = null;
+        // TODO: make static final constant
+
+
+        boolean wasSplitted = false;
+        Float splitX1 = null;
+        for (TextPositionSequence word : textPositions) {
+
+            Matcher listIdentifierPattern = listIdentifier.matcher(word.toString());
+
+            boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+            boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());
+            boolean positiveXGapInline = prev != null && maxX + X_GAP_SPLIT_CONSTANT < word.getMinXDirAdj() && sameLine;
+            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < NEGATIVE_X_GAP_THRESHOLD;
+            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
+            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
+            boolean splitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
+            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
+            boolean fontChange = prev != null && (!word.getFont().equals(prev.getFont()) || !word.getFontStyle()
+                    .equals(prev.getFontStyle()) || word.getFontSize() != prev.getFontSize());
+            boolean newline = prev != null && Math.abs(word.getMinYDirAdj() - prev.getMinYDirAdj()) > word.getHeight();
+            boolean isListIdentifier = listIdentifierPattern.matches();
+
+            if (prev != null && (prev.isParagraphStart() || negativeXGap || positiveXGapInline || yGap || startFromTop || splitByRuling || (newline && (fontChange || isListIdentifier)))) {
+//            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
+
+                Orientation prevOrientation = null;
+                if (!classificationTextBlocks.isEmpty()) {
+                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - X_ALIGNMENT_THRESHOLD).getOrientation();
+                }
+
+                TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);
+
+                classificationTextBlocks.add(classificationTextBlock);
+                wordClusterToCombine = new ArrayList<>();
+
+                if (positiveXGapInline && !splitByRuling) {
+                    wasSplitted = true;
+                    classificationTextBlock.setOrientation(Orientation.LEFT);
+                    splitX1 = word.getMinXDirAdj();
+                } else if (newLineAfterSplit && !splitByRuling) {
+                    wasSplitted = false;
+                    classificationTextBlock.setOrientation(Orientation.RIGHT);
+                    splitX1 = null;
+                } else if (prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (yGap || !startFromTop || !positiveXGapInline || !newLineAfterSplit || !splitByRuling)) {
+                    classificationTextBlock.setOrientation(Orientation.LEFT);
+                }
+
+                minX = 1000;
+                maxX = 0;
+                minY = 1000;
+                maxY = 0;
+                prev = null;
+            }
+
+            wordClusterToCombine.add(word);
+
+            prev = word;
+            if (word.getMinXDirAdj() < minX) {
+                minX = word.getMinXDirAdj();
+            }
+            if (word.getMaxXDirAdj() > maxX) {
+                maxX = word.getMaxXDirAdj();
+            }
+            if (word.getMinYDirAdj() < minY) {
+                minY = word.getMinYDirAdj();
+            }
+            if (word.getMaxYDirAdj() > maxY) {
+                maxY = word.getMaxYDirAdj();
+            }
+        }
+
+        TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);
+        if (classificationTextBlock != null) {
+            classificationTextBlocks.add(classificationTextBlock);
+        }
+        return classificationTextBlocks;
+    }
+
+
+    private boolean equalsWithThreshold(float f1, float f2) {
+
+        return Math.abs(f1 - f2) < THRESHOLD;
+    }
+
+
+    private boolean isSplitByRuling(float minX,
+                                    float minY,
+                                    float maxX,
+                                    float maxY,
+                                    TextPositionSequence word,
+                                    List<Ruling> horizontalRulingLines,
+                                    List<Ruling> verticalRulingLines) {
+
+        return isSplitByRuling(maxX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMinYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(maxX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMinYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()); //
+    }
+
+
+    private boolean isSplitByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines, float dir, float pageWidth, float pageHeight) {
+
+        for (Ruling ruling : rulingLines) {
+            var line = RulingTextDirAdjustUtil.convertToDirAdj(ruling, dir, pageWidth, pageHeight);
+            if (line.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}
+

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/RedactManagerClassificationService.java

@@ -3,15 +3,14 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;
 
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 
 import lombok.RequiredArgsConstructor;
@@ -22,6 +21,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactManagerClassificationService {
 
+
     public void classifyDocument(ClassificationDocument document) {
 
         List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
@@ -52,25 +52,14 @@ public class RedactManagerClassificationService {
             textBlock.setClassification(PageBlockType.OTHER);
             return;
         }
-
-        if (page.getImages()
-                .stream()
-                .filter(image -> image.getImageType().equals(ImageType.GRAPHIC))
-                .anyMatch(graphic -> graphic.getPosition().intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()))) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
-            return;
-        }
-
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
-                textBlock,
-                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
             textBlock.setClassification(PageBlockType.HEADER);
 
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
-                textBlock,
-                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
             textBlock.setClassification(PageBlockType.FOOTER);
         } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                 document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/TaasClassificationService.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;
 
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -10,7 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 
 import lombok.RequiredArgsConstructor;
@@ -19,21 +20,26 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Service
 @RequiredArgsConstructor
-public class ClarifyndClassificationService {
+public class TaasClassificationService {
+
+    private final BodyTextFrameService bodyTextFrameService;
+
 
     public void classifyDocument(ClassificationDocument document) {
 
+
         List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
         for (ClassificationPage page : document.getPages()) {
+
             classifyPage(page, document, headlineFontSizes);
         }
     }
 
 
-    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    public void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
@@ -43,25 +49,20 @@ public class ClarifyndClassificationService {
     }
 
 
-    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    public void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
         if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
+            textBlock.setClassification(PageBlockType.OTHER);
             return;
         }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
-                textBlock,
-                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
-
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
-                textBlock,
-                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
-                .getMostPopular())) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+            textBlock.setClassification(PageBlockType.HEADER);
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+            textBlock.setClassification(PageBlockType.FOOTER);
         } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                 document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
                 .size() == 1)) {
@@ -106,9 +107,8 @@ public class ClarifyndClassificationService {
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
             textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
         } else {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
+            textBlock.setClassification(PageBlockType.OTHER);
         }
     }
 
 }
-

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/DocstrumSegmentationService.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -6,15 +6,15 @@ import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.LineBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.NearestNeighbourService;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.ReadingOrderService;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.SpacingService;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.ZoneBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.LineBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.NearestNeighbourService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ReadingOrderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.SpacingService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ZoneBuilderService;
 
 import lombok.RequiredArgsConstructor;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/AngleFilter.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
 
 public class AngleFilter {
 
@@ -7,7 +7,7 @@ public class AngleFilter {
 
 
     public AngleFilter(double lowerAngle, double upperAngle) {
-
+        
         this.lowerAngle = lowerAngle < -Math.PI / 2 ? lowerAngle + Math.PI : lowerAngle;
         this.upperAngle = upperAngle >= Math.PI / 2 ? upperAngle - Math.PI : upperAngle;
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/BoundingBox.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
 
 import java.awt.geom.Rectangle2D;
 
@@ -42,7 +42,10 @@ public abstract class BoundingBox {
 
     public boolean contains(Rectangle2D contained, double tolerance) {
 
-        return bBox.getX() <= contained.getX() + tolerance && bBox.getY() <= contained.getY() + tolerance && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
+        return bBox.getX() <= contained.getX() + tolerance
+               && bBox.getY() <= contained.getY() + tolerance
+               && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance
+               && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Character.java

@@ -1,27 +1,20 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 
 import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.Setter;
 
 @Data
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Character {
 
-    @EqualsAndHashCode.Include
     private final double x;
-    @EqualsAndHashCode.Include
     private final double y;
     private final RedTextPosition textPosition;
 
-    @Setter
     private List<Neighbor> neighbors = new ArrayList<>();
 
 
@@ -74,12 +67,18 @@ public class Character {
     }
 
 
+    public void setNeighbors(List<Neighbor> neighbors) {
+
+        this.neighbors = neighbors;
+    }
+
+
     public double angle(Character character) {
 
         if (getX() > character.getX()) {
-            return FastAtan2.fastAtan2(getY() - character.getY(), getX() - character.getX());
+            return Math.atan2(getY() - character.getY(), getX() - character.getX());
         } else {
-            return FastAtan2.fastAtan2(character.getY() - getY(), character.getX() - getX());
+            return Math.atan2(character.getY() - getY(), character.getX() - getX());
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/DisjointSets.java

@@ -0,0 +1,194 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.util.AbstractSet;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+public class DisjointSets<E> implements Iterable<Set<E>> {
+
+    private final Map<E, Entry<E>> map = new HashMap<>();
+
+
+    public DisjointSets(Collection<? extends E> collection) {
+
+        for (E element : collection) {
+            map.put(element, new Entry<E>(element));
+        }
+    }
+
+
+    public boolean areTogether(E e1, E e2) {
+
+        return map.get(e1).findRepresentative().equals(map.get(e2).findRepresentative());
+    }
+
+
+    public void union(E e1, E e2) {
+
+        Entry<E> r1 = map.get(e1).findRepresentative();
+        Entry<E> r2 = map.get(e2).findRepresentative();
+        if (!r1.equals(r2)) {
+            if (r1.size <= r2.size) {
+                r2.mergeWith(r1);
+            } else {
+                r1.mergeWith(r2);
+            }
+        }
+    }
+
+
+    @Override
+    public Iterator<Set<E>> iterator() {
+
+        return new Iterator<>() {
+
+            private final Iterator<Entry<E>> iterator = map.values().iterator();
+            private Entry<E> nextRepresentative;
+
+            {
+                findNextRepresentative();
+            }
+
+            @Override
+            public boolean hasNext() {
+
+                return nextRepresentative != null;
+            }
+
+
+            @Override
+            public Set<E> next() {
+
+                if (nextRepresentative == null) {
+                    throw new NoSuchElementException();
+                }
+                Set<E> result = nextRepresentative.asSet();
+                findNextRepresentative();
+                return result;
+            }
+
+
+            private void findNextRepresentative() {
+
+                while (iterator.hasNext()) {
+                    Entry<E> candidate = iterator.next();
+                    if (candidate.isRepresentative()) {
+                        nextRepresentative = candidate;
+                        return;
+                    }
+                }
+                nextRepresentative = null;
+            }
+
+
+            @Override
+            public void remove() {
+
+                throw new UnsupportedOperationException();
+            }
+
+        };
+    }
+
+
+    private static class Entry<E> {
+
+        private int size = 1;
+        private final E value;
+        private Entry<E> parent = this;
+        private Entry<E> next;
+        private Entry<E> last = this;
+
+
+        Entry(E value) {
+
+            this.value = value;
+        }
+
+
+        void mergeWith(Entry<E> otherRepresentative) {
+
+            size += otherRepresentative.size;
+            last.next = otherRepresentative;
+            last = otherRepresentative.last;
+            otherRepresentative.parent = this;
+        }
+
+
+        Entry<E> findRepresentative() {
+
+            Entry<E> representative = parent;
+            while (!representative.parent.equals(representative)) {
+                representative = representative.parent;
+            }
+            for (Entry<E> entry = this; !entry.equals(representative); ) {
+                Entry<E> nextEntry = entry.parent;
+                entry.parent = representative;
+                entry = nextEntry;
+            }
+            return representative;
+        }
+
+
+        boolean isRepresentative() {
+
+            return parent.equals(this);
+        }
+
+
+        Set<E> asSet() {
+
+            return new AbstractSet<E>() {
+
+                @Override
+                public Iterator<E> iterator() {
+
+                    return new Iterator<E>() {
+
+                        private Entry<E> nextEntry = findRepresentative();
+
+
+                        @Override
+                        public boolean hasNext() {
+
+                            return nextEntry != null;
+                        }
+
+
+                        @Override
+                        public E next() {
+
+                            if (nextEntry == null) {
+                                throw new NoSuchElementException();
+                            }
+                            E result = nextEntry.value;
+                            nextEntry = nextEntry.next;
+                            return result;
+                        }
+
+
+                        @Override
+                        public void remove() {
+
+                            throw new UnsupportedOperationException();
+                        }
+
+                    };
+                }
+
+
+                @Override
+                public int size() {
+
+                    return findRepresentative().size;
+                }
+            };
+        }
+
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Histogram.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
 
 public class Histogram {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Line.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
 
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
@@ -8,22 +8,16 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.Data;
-import lombok.EqualsAndHashCode;
 
 @Data
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Line extends BoundingBox {
 
     private static final double WORD_DISTANCE_MULTIPLIER = 0.18;
 
-    @EqualsAndHashCode.Include
     private final double x0;
-    @EqualsAndHashCode.Include
     private final double y0;
 
-    @EqualsAndHashCode.Include
     private final double x1;
-    @EqualsAndHashCode.Include
     private final double y1;
 
     private final double height;
@@ -84,7 +78,11 @@ public class Line extends BoundingBox {
 
     private double computeHeight() {
 
-        return characters.stream().map(Character::getHeight).reduce(0d, Double::sum) / characters.size();
+        double sum = 0.0;
+        for (Character component : characters) {
+            sum += component.getHeight();
+        }
+        return sum / characters.size();
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Neighbor.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
 
 import lombok.Getter;
 
@@ -6,7 +6,8 @@ public class Neighbor {
 
     @Getter
     private final double distance;
-    private Double angle;
+    @Getter
+    private final double angle;
     private final Character originCharacter;
     @Getter
     private final Character character;
@@ -15,6 +16,7 @@ public class Neighbor {
     public Neighbor(Character neighbor, Character origin) {
 
         this.distance = neighbor.distance(origin);
+        this.angle = neighbor.angle(origin);
         this.character = neighbor;
         this.originCharacter = origin;
     }
@@ -31,13 +33,4 @@ public class Neighbor {
         return character.verticalDistance(originCharacter);
     }
 
-
-    public double getAngle() {
-
-        if (angle != null) {
-            return angle;
-        }
-        return this.character.angle(this.originCharacter);
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Zone.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Comparator;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/LineBuilderService.java

@@ -1,16 +1,15 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.HashSet;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
 
 @Service
 public class LineBuilderService {
@@ -25,8 +24,7 @@ public class LineBuilderService {
         double maxHorizontalDistance = characterSpacing * CHARACTER_SPACING_DISTANCE_MULTIPLIER;
         double maxVerticalDistance = lineSpacing * MAX_VERTICAL_CHARACTER_DISTANCE;
 
-        UnionFind<Character> unionFind = new UnionFind<>(new HashSet<>(characters));
-
+        DisjointSets<Character> sets = new DisjointSets<>(characters);
         AngleFilter filter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
 
         characters.forEach(character -> {
@@ -35,13 +33,13 @@ public class LineBuilderService {
                 double y = neighbor.getVerticalDistance() / maxVerticalDistance;
                 if (character.getTextPosition().getDir() == neighbor.getCharacter().getTextPosition().getDir() && filter.matches(neighbor) && Math.pow(x, 2) + Math.pow(y,
                         2) <= 1) {
-                    unionFind.union(character, neighbor.getCharacter());
+                    sets.union(character, neighbor.getCharacter());
                 }
             });
         });
 
         List<Line> lines = new ArrayList<>();
-        unionFind.getGroups().forEach(group -> {
+        sets.forEach(group -> {
             List<Character> lineCharacters = new ArrayList<>(group);
             lineCharacters.sort(Comparator.comparingDouble(Character::getX));
             lines.add(new Line(lineCharacters, characterSpacing));

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/NearestNeighbourService.java

@@ -0,0 +1,78 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
+
+@Service
+public class NearestNeighbourService {
+
+    private static final int NUMBER_OF_NEIGHBOURS = 8;
+    private static final double STEP = 16.0;
+
+
+    public void findNearestNeighbors(List<Character> characters) {
+
+        if (characters.isEmpty() || characters.size() == 1) {
+            return;
+        }
+
+        characters.sort(Comparator.comparingDouble(Character::getX));
+
+        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
+        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
+            maxNeighborCount = characters.size() - 1;
+        }
+
+        for (int i = 0; i < characters.size(); i++) {
+
+            List<Neighbor> candidates = new ArrayList<>();
+
+            int start = i;
+            int end = i + 1;
+
+            double distance = Double.POSITIVE_INFINITY;
+
+            for (double searchDistance = 0; searchDistance < distance; ) {
+
+                searchDistance += STEP;
+                boolean newCandidatesFound = false;
+
+                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
+                    start--;
+                    candidates.add(new Neighbor(characters.get(start), characters.get(i)));
+                    clearLeastDistant(candidates, maxNeighborCount);
+                    newCandidatesFound = true;
+                }
+
+                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
+                    candidates.add(new Neighbor(characters.get(end), characters.get(i)));
+                    clearLeastDistant(candidates, maxNeighborCount);
+                    end++;
+                    newCandidatesFound = true;
+                }
+
+                if (newCandidatesFound && candidates.size() >= maxNeighborCount) {
+                    distance = candidates.get(maxNeighborCount - 1).getDistance();
+                }
+            }
+            clearLeastDistant(candidates, maxNeighborCount);
+            characters.get(i).setNeighbors(new ArrayList<>(candidates));
+        }
+    }
+
+
+    private void clearLeastDistant(List<Neighbor> candidates, int maxNeighborCount) {
+
+        if (candidates.size() > maxNeighborCount) {
+            candidates.sort(Comparator.comparingDouble(Neighbor::getDistance));
+            candidates.remove(candidates.remove(candidates.size() - 1));
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/ReadingOrderService.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -9,9 +9,9 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
 
 @Service
 public class ReadingOrderService {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/SpacingService.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
 
 import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Histogram;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Histogram;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
 
 @Service
 public class SpacingService {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/ZoneBuilderService.java

@@ -1,17 +1,16 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
 
 @Service
 public class ZoneBuilderService {
@@ -41,7 +40,7 @@ public class ZoneBuilderService {
         double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
         double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;
 
-        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
+        DisjointSets<Line> sets = new DisjointSets<>(lines);
 
         double meanHeight = calculateMeanHeight(lines);
 
@@ -51,21 +50,21 @@ public class ZoneBuilderService {
                     double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
                     scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
 
-                    if (!unionFind.inSameSet(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {
+                    if (!sets.areTogether(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {
 
                         double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
                         double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
 
                         if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance //
                                 || minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
-                            unionFind.union(outerLine, innerLine);
+                            sets.union(outerLine, innerLine);
                         }
                     }
                 }));
 
         List<Zone> zones = new ArrayList<>();
-        unionFind.getGroups().forEach(group -> {
-            zones.add(mergeLinesInZone(new ArrayList<>(group), characterSpacing, lineSpacing));
+        sets.forEach(group -> {
+            zones.add(new Zone(new ArrayList<>(group)));
         });
 
         if (zones.size() > MAX_ZONES) {
@@ -100,7 +99,7 @@ public class ZoneBuilderService {
         double minVerticalDistance = 0;
         double maxVerticalDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE;
 
-        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
+        DisjointSets<Line> sets = new DisjointSets<>(lines);
 
         lines.forEach(outer -> {
 
@@ -111,7 +110,7 @@ public class ZoneBuilderService {
                     double verticalDistance = outer.verticalDistance(inner);
 
                     if (horizontalDistance <= maxHorizontalDistance && minVerticalDistance <= verticalDistance && verticalDistance <= maxVerticalDistance) {
-                        unionFind.union(outer, inner);
+                        sets.union(outer, inner);
                     } else if (minVerticalDistance <= verticalDistance && verticalDistance <= maxVerticalDistance && Math.abs(horizontalDistance - Math.min(outer.getLength(),
                             inner.getLength())) < 0.1) {
                         boolean characterOverlap = false;
@@ -128,7 +127,7 @@ public class ZoneBuilderService {
                             }
                         }
                         if (!characterOverlap && overlappingCount <= 2) {
-                            unionFind.union(outer, inner);
+                            sets.union(outer, inner);
                         }
                     }
                 }
@@ -136,16 +135,15 @@ public class ZoneBuilderService {
         });
 
         List<Line> outputZone = new ArrayList<>();
-        for (Set<Line> group : unionFind.getGroups()) {
-            List<Character> characters = new ArrayList<>();
+        for (Set<Line> group : sets) {
+            List<Character> components = new ArrayList<>();
             for (Line line : group) {
-                characters.addAll(line.getCharacters());
+                components.addAll(line.getCharacters());
             }
-            characters.sort(Comparator.comparingDouble(Character::getX));
+            components.sort(Comparator.comparingDouble(Character::getX));
 
-            outputZone.add(new Line(characters, characterSpacing));
+            outputZone.add(new Line(components, characterSpacing));
         }
-
         return new Zone(outputZone);
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/utils/DoubleUtils.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.docstrum.utils;
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils;
 
 public class DoubleUtils {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java

@@ -16,7 +16,6 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
@@ -49,19 +48,14 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class DocumentGraphFactory {
 
-    public Document buildDocumentGraph(LayoutParsingType layoutParsingType, ClassificationDocument document) {
+    public Document buildDocumentGraph(ClassificationDocument document) {
 
         Document documentGraph = new Document();
         Context context = new Context(documentGraph);
 
-        document.getPages()
-                .forEach(context::buildAndAddPageWithCounter);
-        document.getSections()
-                .stream()
-                .flatMap(section -> section.getImages()
-                        .stream())
-                .forEach(image -> context.getImages().add(image));
-        addSections(layoutParsingType, document, context, documentGraph);
+        document.getPages().forEach(context::buildAndAddPageWithCounter);
+        document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.getImages().add(image));
+        addSections(document, context);
         addHeaderAndFooterToEachPage(document, context);
 
         documentGraph.setNumberOfPages(context.pages.size());
@@ -72,10 +66,9 @@ public class DocumentGraphFactory {
     }
 
 
-    private void addSections(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument, Context context, Document document) {
+    private void addSections(ClassificationDocument document, Context context) {
 
-        classificationDocument.getSections()
-                .forEach(section -> SectionNodeFactory.addSection(layoutParsingType, null, section.getNonEmptyPageBlocks(), section.getImages(), context, document));
+        document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getNonEmptyPageBlocks(), section.getImages(), context));
     }
 
 
@@ -85,14 +78,11 @@ public class DocumentGraphFactory {
 
         GenericSemanticNode node;
         if (originalTextBlock.isHeadline()) {
-            node = Headline.builder().documentTree(context.getDocumentTree())
-                    .build();
+            node = Headline.builder().documentTree(context.getDocumentTree()).build();
         } else if (originalTextBlock.isToDuplicate()) {
-            node = DuplicatedParagraph.builder().documentTree(context.getDocumentTree())
-                    .build();
+            node = DuplicatedParagraph.builder().documentTree(context.getDocumentTree()).build();
         } else {
-            node = Paragraph.builder().documentTree(context.getDocumentTree())
-                    .build();
+            node = Paragraph.builder().documentTree(context.getDocumentTree()).build();
         }
 
         page.getMainBody().add(node);
@@ -100,14 +90,12 @@ public class DocumentGraphFactory {
         List<TextPageBlock> textBlocks = new ArrayList<>();
         textBlocks.add(originalTextBlock);
         textBlocks.addAll(textBlocksToMerge);
-
         AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
 
         if (node instanceof DuplicatedParagraph duplicatedParagraph) {
             AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock(textBlocks.stream()
-                                                                                                      .flatMap(tb -> tb.getSequences()
-                                                                                                              .stream())
-                                                                                                      .collect(Collectors.toList()), node, context, page);
+                    .flatMap(tb -> tb.getSequences().stream())
+                    .collect(Collectors.toList()), node, context, page);
             duplicatedParagraph.setUnsortedLeafTextBlock(unsortedTextBlock);
         }
 
@@ -117,29 +105,25 @@ public class DocumentGraphFactory {
     }
 
 
-    public void addImage(GenericSemanticNode parent, ClassifiedImage image, Context context) {
-
-        Image imageNode = createImage(image, context);
-        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parent, imageNode);
-        imageNode.setTreeId(treeId);
-        imageNode.setLeafTextBlock(context.textBlockFactory.emptyTextBlock(parent, context, context.getPage(image.getPage())));
-    }
-
-
-    private Image createImage(ClassifiedImage image, Context context) {
+    public void addImage(Section section, ClassifiedImage image, Context context) {
 
         Rectangle2D position = image.getPosition();
         Page page = context.getPage(image.getPage());
-        Image imageNode = Image.builder()
+        var imageBuilder = Image.builder()
                 .id(IdBuilder.buildId(Set.of(page), List.of(position)))
                 .imageType(image.getImageType())
                 .position(position)
                 .transparent(image.isHasTransparency())
                 .page(page)
-                .documentTree(context.getDocumentTree())
-                .build();
+                .documentTree(context.getDocumentTree());
+        if (image.isSourceByAi()) {
+            imageBuilder.engines(new HashSet<>(Set.of(LayoutEngine.AI)));
+        }
+        Image imageNode = imageBuilder.build();
         page.getMainBody().add(imageNode);
-        return imageNode;
+
+        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode);
+        imageNode.setTreeId(treeId);
     }
 
 
@@ -178,12 +162,11 @@ public class DocumentGraphFactory {
     private void addFooter(List<TextPageBlock> textBlocks, Context context) {
 
         Page page = context.getPage(textBlocks.get(0).getPage());
-        Footer footer = Footer.builder().documentTree(context.getDocumentTree())
-                .build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeTextPositionSequence(textBlocks),
-                                                                                  footer,
-                                                                                  context,
-                                                                                  page);
+        Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build();
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
+                footer,
+                context,
+                page);
         List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer);
         footer.setTreeId(tocId);
         footer.setLeafTextBlock(textBlock);
@@ -194,9 +177,8 @@ public class DocumentGraphFactory {
     public void addHeader(List<TextPageBlock> textBlocks, Context context) {
 
         Page page = context.getPage(textBlocks.get(0).getPage());
-        Header header = Header.builder().documentTree(context.getDocumentTree())
-                .build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeTextPositionSequence(textBlocks), header, 0, page);
+        Header header = Header.builder().documentTree(context.getDocumentTree()).build();
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, 0, page);
         List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header);
         header.setTreeId(tocId);
         header.setLeafTextBlock(textBlock);
@@ -207,8 +189,7 @@ public class DocumentGraphFactory {
     private void addEmptyFooter(int pageIndex, Context context) {
 
         Page page = context.getPage(pageIndex);
-        Footer footer = Footer.builder().documentTree(context.getDocumentTree())
-                .build();
+        Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build();
         AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page);
         List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer);
         footer.setTreeId(tocId);
@@ -220,8 +201,7 @@ public class DocumentGraphFactory {
     private void addEmptyHeader(int pageIndex, Context context) {
 
         Page page = context.getPage(pageIndex);
-        Header header = Header.builder().documentTree(context.getDocumentTree())
-                .build();
+        Header header = Header.builder().documentTree(context.getDocumentTree()).build();
         AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
         List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header);
         header.setTreeId(tocId);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SearchTextWithTextPositionFactory.java

@@ -8,10 +8,10 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
 
 import lombok.experimental.UtilityClass;
 
@@ -110,7 +110,6 @@ public class SearchTextWithTextPositionFactory {
         return context.stringIdx - context.lastHyphenIdx < MAX_HYPHEN_LINEBREAK_DISTANCE;
     }
 
-
     private static List<Boundary> mergeToBoundaries(List<Integer> integers) {
 
         if (integers.isEmpty()) {
@@ -126,9 +125,8 @@ public class SearchTextWithTextPositionFactory {
             }
             end = current + 1;
         }
-        if (boundaries.isEmpty()) {
+        if (boundaries.isEmpty())
             boundaries.add(new Boundary(start, end));
-        }
         return boundaries;
     }
 
@@ -140,7 +138,6 @@ public class SearchTextWithTextPositionFactory {
         }
     }
 
-
     private boolean isLineBreak(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition) {
 
         return Objects.equals(currentTextPosition.getUnicode(), "\n") || isDeltaYLargerThanTextHeight(currentTextPosition, previousTextPosition);
@@ -180,7 +177,7 @@ public class SearchTextWithTextPositionFactory {
     }
 
 
-    public Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
+    private Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
 
         float textHeight = sequence.getTextHeight() + HEIGHT_PADDING;
         Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(),

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SectionNodeFactory.java

@@ -4,16 +4,13 @@ import static java.lang.String.format;
 import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.groupingBy;
 
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
@@ -27,52 +24,27 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class SectionNodeFactory {
 
-    public void addSection(LayoutParsingType layoutParsingType,
-                           GenericSemanticNode parentNode,
-                           List<AbstractPageBlock> pageBlocks,
-                           List<ClassifiedImage> images,
-                           DocumentGraphFactory.Context context,
-                           Document document) {
-
-        // This is for the case where we have images on a page without any text/footer/header.
-        // The pageBlocks list is empty, but we still need to add those images to the document.
-        if (!images.isEmpty() && pageBlocks.isEmpty()) {
-            images.stream()
-                    .distinct()
-                    .forEach(image -> DocumentGraphFactory.addImage(document, image, context));
-            return;
-        }
+    public void addSection(GenericSemanticNode parentNode, List<AbstractPageBlock> pageBlocks, List<ClassifiedImage> images, DocumentGraphFactory.Context context) {
 
         if (pageBlocks.isEmpty()) {
             return;
         }
-
-        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream()
-                .collect(groupingBy(AbstractPageBlock::getPage));
-        Section section = Section.builder().documentTree(context.getDocumentTree())
-                .build();
+        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
+        Section section = Section.builder().documentTree(context.getDocumentTree()).build();
 
         context.getSections().add(section);
-        blocksPerPage.keySet()
-                .forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));
+        blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));
 
         section.setTreeId(getTreeId(parentNode, context, section));
 
-        addFirstHeadlineDirectlyToSection(layoutParsingType, pageBlocks, context, section, document);
+        addFirstHeadlineDirectlyToSection(pageBlocks, context, section);
         if (containsTablesAndTextBlocks(pageBlocks)) {
-            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(layoutParsingType,
-                                                                                                  section,
-                                                                                                  subSectionPageBlocks,
-                                                                                                  emptyList(),
-                                                                                                  context,
-                                                                                                  document));
+            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(section, subSectionPageBlocks, emptyList(), context));
         } else {
-            addTablesAndParagraphsAndHeadlinesToSection(layoutParsingType, pageBlocks, context, section, document);
+            addTablesAndParagraphsAndHeadlinesToSection(pageBlocks, context, section);
         }
 
-        images.stream()
-                .distinct()
-                .forEach(image -> DocumentGraphFactory.addImage(section, image, context));
+        images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(section, image, context));
     }
 
 
@@ -86,24 +58,16 @@ public class SectionNodeFactory {
     }
 
 
-    private void addFirstHeadlineDirectlyToSection(LayoutParsingType layoutParsingType,
-                                                   List<AbstractPageBlock> pageBlocks,
-                                                   DocumentGraphFactory.Context context,
-                                                   Section section,
-                                                   Document document) {
+    private void addFirstHeadlineDirectlyToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
 
         if (pageBlocks.get(0).isHeadline()) {
-            addTablesAndParagraphsAndHeadlinesToSection(layoutParsingType, List.of(pageBlocks.get(0)), context, section, document);
+            addTablesAndParagraphsAndHeadlinesToSection(List.of(pageBlocks.get(0)), context, section);
             pageBlocks.remove(0);
         }
     }
 
 
-    private void addTablesAndParagraphsAndHeadlinesToSection(LayoutParsingType layoutParsingType,
-                                                             List<AbstractPageBlock> pageBlocks,
-                                                             DocumentGraphFactory.Context context,
-                                                             Section section,
-                                                             Document document) {
+    private void addTablesAndParagraphsAndHeadlinesToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
 
         Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
         List<AbstractPageBlock> remainingBlocks = new LinkedList<>(pageBlocks);
@@ -116,23 +80,13 @@ public class SectionNodeFactory {
             remainingBlocks.removeAll(alreadyMerged);
 
             if (abstractPageBlock instanceof TextPageBlock) {
-
-                switch (layoutParsingType) {
-                    case REDACT_MANAGER, CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> {
-                        alreadyMerged.add(abstractPageBlock);
-                        remainingBlocks.remove(abstractPageBlock);
-                        DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, new ArrayList<>());
-                    }
-                    default -> {
-                        List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY((TextPageBlock) abstractPageBlock, remainingBlocks);
-                        alreadyMerged.addAll(textBlocks);
-                        DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
-                    }
-                }
+                List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY((TextPageBlock) abstractPageBlock, remainingBlocks);
+                alreadyMerged.addAll(textBlocks);
+                DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
             } else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
                 List<TablePageBlock> tablesToMerge = TableMergingUtility.findConsecutiveTablesWithSameColCountAndSameHeaders(tablePageBlock, remainingBlocks);
                 alreadyMerged.addAll(tablesToMerge);
-                TableNodeFactory.addTable(layoutParsingType, section, tablesToMerge, context, document);
+                TableNodeFactory.addTable(section, tablesToMerge, context);
             } else {
                 throw new RuntimeException(format("Unhandled AbstractPageBlockType %s!", abstractPageBlock.getClass()));
             }
@@ -142,9 +96,7 @@ public class SectionNodeFactory {
 
     private boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {
 
-        return pageBlocks.stream()
-                       .anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
-                       .anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
+        return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
     }
 
 
@@ -160,9 +112,7 @@ public class SectionNodeFactory {
 
         List<List<AbstractPageBlock>> splitList = splitIntoCoherentList(pageBlocks);
         movePrecedingHeadlineToTableList(splitList);
-        return splitList.stream()
-                .filter(list -> !list.isEmpty())
-                .toList();
+        return splitList.stream().filter(list -> !list.isEmpty()).toList();
     }
 
 
@@ -183,8 +133,7 @@ public class SectionNodeFactory {
 
     private boolean listIsTablesOnly(List<AbstractPageBlock> abstractPageBlocks) {
 
-        return abstractPageBlocks.stream()
-                .allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock);
+        return abstractPageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/TableNodeFactory.java

@@ -7,18 +7,16 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
 import lombok.experimental.UtilityClass;
@@ -29,30 +27,23 @@ public class TableNodeFactory {
     public final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
 
 
-    public void addTable(LayoutParsingType layoutParsingType,
-                         GenericSemanticNode parentNode,
-                         List<TablePageBlock> tablesToMerge,
-                         DocumentGraphFactory.Context context,
-                         Document document) {
+    public void addTable(GenericSemanticNode parentNode, List<TablePageBlock> tablesToMerge, DocumentGraphFactory.Context context) {
 
         setPageNumberInCells(tablesToMerge);
-        Set<Page> pages = tablesToMerge.stream()
-                .map(AbstractPageBlock::getPage)
-                .map(context::getPage)
-                .collect(Collectors.toSet());
-        List<List<Cell>> mergedRows = tablesToMerge.stream()
-                .map(TablePageBlock::getRows)
-                .flatMap(Collection::stream)
-                .toList();
+        Set<Page> pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet());
+        List<List<Cell>> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList();
 
-        Table table = Table.builder().documentTree(context.getDocumentTree()).numberOfCols(mergedRows.isEmpty() ? 0 : mergedRows.get(0).size()).numberOfRows(mergedRows.size())
+        Table table = Table.builder()
+                .documentTree(context.getDocumentTree())
+                .numberOfCols(mergedRows.isEmpty() ? 0 : mergedRows.get(0).size())
+                .numberOfRows(mergedRows.size())
                 .build();
 
         pages.forEach(page -> addTableToPage(page, parentNode, table));
 
         List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table);
         table.setTreeId(treeId);
-        addTableCells(layoutParsingType, mergedRows, table, context, document);
+        addTableCells(mergedRows, table, context);
 
         ifTableHasNoHeadersSetFirstRowAsHeaders(table);
     }
@@ -72,8 +63,7 @@ public class TableNodeFactory {
 
     private void setPageNumberInTextBlocksWithPageNumberSetTo0(TablePageBlock table, Cell cell) {
 
-        cell.getTextBlocks()
-                .stream()//
+        cell.getTextBlocks().stream()//
                 .filter(tb -> tb.getPage() == 0)//
                 .forEach(tb -> tb.setPage(table.getPage()));
     }
@@ -92,44 +82,28 @@ public class TableNodeFactory {
 
     private void ifTableHasNoHeadersSetFirstRowAsHeaders(Table table) {
 
-        if (table.streamHeaders()
-                .findAny().isEmpty()) {
-            table.streamRow(0)
-                    .forEach(tableCellNode -> tableCellNode.setHeader(true));
+        if (table.streamHeaders().findAny().isEmpty()) {
+            table.streamRow(0).forEach(tableCellNode -> tableCellNode.setHeader(true));
         }
     }
 
 
-    private void addTableCells(LayoutParsingType layoutParsingType, List<List<Cell>> rows, Table table, DocumentGraphFactory.Context context, Document document) {
+    private void addTableCells(List<List<Cell>> rows, Table table, DocumentGraphFactory.Context context) {
 
         for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
             for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(layoutParsingType,
-                             rows.get(rowIndex)
-                                     .get(colIndex),
-                             rowIndex,
-                             colIndex,
-                             table,
-                             context,
-                             document);
+                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, table, context);
             }
         }
     }
 
 
     @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong
-    private void addTableCell(LayoutParsingType layoutParsingType,
-                              Cell cell,
-                              int rowIndex,
-                              int colIndex,
-                              Table tableNode,
-                              DocumentGraphFactory.Context context,
-                              Document document) {
+    private void addTableCell(Cell cell, int rowIndex, int colIndex, Table tableNode, DocumentGraphFactory.Context context) {
 
         Page page = context.getPage(cell.getPageNumber());
 
-        TableCell tableCell = TableCell.builder().documentTree(context.getDocumentTree()).row(rowIndex).col(colIndex).header(cell.isHeaderCell()).bBox(cell.getBounds2D())
-                .build();
+        TableCell tableCell = TableCell.builder().documentTree(context.getDocumentTree()).row(rowIndex).col(colIndex).header(cell.isHeaderCell()).bBox(cell.getBounds2D()).build();
         page.getMainBody().add(tableCell);
 
         List<Integer> treeId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell);
@@ -139,27 +113,16 @@ public class TableNodeFactory {
         if (cell.getTextBlocks().isEmpty()) {
             tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page));
         } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.getTextBlockFactory()
-                    .buildAtomicTextBlock(cell.getTextBlocks()
-                                                  .get(0).getSequences(), tableCell, context, page);
+            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
         } else if (firstTextBlockIsHeadline(cell)) {
-            SectionNodeFactory.addSection(layoutParsingType,
-                                          tableCell,
-                                          cell.getTextBlocks()
-                                                  .stream()
-                                                  .map(tb -> (AbstractPageBlock) tb)
-                                                  .toList(),
-                                          emptyList(),
-                                          context,
-                                          document);
+            SectionNodeFactory.addSection(tableCell, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
         } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
             List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
             textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
         } else {
-            cell.getTextBlocks()
-                    .forEach(tb -> DocumentGraphFactory.addParagraphOrHeadline(tableCell, tb, context, emptyList()));
+            cell.getTextBlocks().forEach(tb -> DocumentGraphFactory.addParagraphOrHeadline(tableCell, tb, context, emptyList()));
         }
     }
 
@@ -172,8 +135,7 @@ public class TableNodeFactory {
 
     private boolean firstTextBlockIsHeadline(Cell cell) {
 
-        return cell.getTextBlocks()
-                .get(0).isHeadline();
+        return cell.getTextBlocks().get(0).isHeadline();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/Box.java

@@ -1,162 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.graphics;
-
-import java.awt.geom.AffineTransform;
-import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
-import java.util.List;
-import java.util.Optional;
-
-public class Box {
-
-    public double x1;
-    public double y1;
-    public double x2;
-    public double y2;
-
-
-    public Box(double x1, double y1, double x2, double y2) {
-
-        this.x1 = x1;
-        this.y1 = y1;
-        this.x2 = x2;
-        this.y2 = y2;
-    }
-
-
-    public Box(Rectangle2D rectangle2D) {
-
-        this.x1 = rectangle2D.getMinX();
-        this.y1 = rectangle2D.getMinY();
-        this.x2 = rectangle2D.getMaxX();
-        this.y2 = rectangle2D.getMaxY();
-    }
-
-
-    public double width() {
-
-        return x2 - x1;
-    }
-
-
-    public double height() {
-
-        return y2 - y1;
-    }
-
-
-    public double xCenter() {
-
-        return (x2 + x1) / 2;
-    }
-
-
-    public double yCenter() {
-
-        return (y2 + y1) / 2;
-    }
-
-
-    public double area() {
-
-        return width() * height();
-    }
-
-
-    public Box scale(double scale) {
-
-        return new Box(x1 * scale, y1 * scale, x2 * scale, y2 * scale);
-    }
-
-
-    public boolean horizontallyAligned(Box other, double tol) {
-
-        return !(other.x1 - tol > x2 || other.x2 + tol < x1);
-    }
-
-
-    public double yDistanceTo(Box other) {
-
-        return Math.min(Math.abs(other.y1 - y2), Math.abs(y2 - other.y1));
-    }
-
-
-    public boolean intersects(Box other, double tol) {
-
-        return !((x2 < other.x1 - tol) || (x1 > other.x2 + tol) || (y2 < other.y1 - tol) || (y1 > other.y2 + tol));
-    }
-
-
-    public boolean intersectsAndOver(Box other, double tol) {
-
-        return (!((x2 < other.x1 - tol) || (x1 > other.x2 + tol) || (y2 < other.y1 - tol) || (y1 > other.y2 + tol))) && other.y1 > y1;
-    }
-
-
-    public boolean intersectsCenter(Box other, double tol) {
-
-        return !((x2 < other.xCenter() - tol) || (x1 > other.xCenter() + tol) || (y2 < other.yCenter() - tol) || (y1 > other.yCenter() + tol));
-    }
-
-
-    public Optional<Box> intersectRegion(Box other, double tol) {
-
-        if (!intersects(other, tol)) {
-            return Optional.empty();
-        } else {
-            var overlapX1 = Math.max(x1, other.x1);
-            var overlapY1 = Math.max(y1, other.y1);
-            var overlapX2 = Math.min(x2, other.x2);
-            var overlapY2 = Math.min(y2, other.y2);
-            return Optional.of(new Box(overlapX1, overlapY1, overlapX2, overlapY2));
-        }
-    }
-
-
-    public double intersectArea(Box other, double tol) {
-
-        return intersectRegion(other, tol).map(Box::area).orElse(0d);
-    }
-
-
-    public boolean intersectsAny(List<Box> others, double tol) {
-
-        return others.stream().anyMatch(other -> intersects(other, tol));
-    }
-
-
-    public boolean intersectsAnyAndOver(List<Box> others, double tol) {
-
-        return others.stream().anyMatch(other -> intersectsAndOver(other, tol));
-    }
-
-
-    public boolean intersectsCenter(List<Box> others, double tol) {
-
-        return others.stream().anyMatch(other -> intersectsCenter(other, tol));
-    }
-
-
-    public boolean contains(Box other, double tol) {
-
-        return (x1 <= other.x1 + tol) && (y1 <= other.y1 + tol) && (x2 >= other.x2 - tol) && (y2 >= other.y2 - tol);
-    }
-
-
-    public Box container(Box other) {
-
-        var minX = Math.min(x1, other.x1);
-        var minY = Math.min(y1, other.y1);
-        var maxX = Math.max(x2, other.x2);
-        var maxY = Math.max(y2, other.y2);
-        return new Box(minX, minY, maxX, maxY);
-    }
-
-
-    public Box transform(AffineTransform affineTransform) {
-
-        Point2D point = affineTransform.transform(new Point2D.Double(x1, y1), null);
-        Point2D point2 = affineTransform.transform(new Point2D.Double(x2, y2), null);
-        return new Box(Math.min(point.getX(), point2.getX()), Math.min(point.getY(), point2.getY()), Math.max(point.getX(), point2.getX()), Math.max(point.getY(), point2.getY()));
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/DistinctQueue.java

@@ -1,51 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.graphics;
-
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.Set;
-
-public class DistinctQueue<T> {
-
-    private Queue<T> queue;
-    private Set<T> set;
-
-
-    public DistinctQueue() {
-
-        queue = new LinkedList<>();
-        set = new HashSet<>();
-    }
-
-
-    public void enqueue(T element) {
-
-        if (!set.contains(element)) {
-            queue.add(element);
-            set.add(element);
-        }
-    }
-
-
-    public T dequeue() {
-
-        T element = queue.poll();
-        if (element != null) {
-            set.remove(element);
-        }
-        return element;
-    }
-
-
-    public boolean isEmpty() {
-
-        return queue.isEmpty();
-    }
-
-
-    public int size() {
-
-        return queue.size();
-    }
-    // Other methods as needed
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/FindGraphicsRaster.java

@@ -1,172 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.graphics;
-
-import java.awt.geom.AffineTransform;
-import java.awt.geom.Rectangle2D;
-import java.awt.image.BufferedImage;
-import java.awt.image.DataBufferByte;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.rendering.ImageType;
-import org.apache.pdfbox.rendering.PDFRenderer;
-import org.springframework.stereotype.Service;
-
-import lombok.SneakyThrows;
-
-@Service
-public class FindGraphicsRaster {
-
-    // Pixels that are lighter then this threshold are ignored
-    private final static int THRESHOLD = 240;
-
-    // DPI to render the image at, in practice sub-72 seems to risk pixels being lost
-    private final static int DPI = 72;
-
-
-    @SneakyThrows
-    public List<Box> findCCBoundingBoxes(PDDocument doc, List<Rectangle2D> remove, PageInformation pageInformation) {
-
-        var renderer = new PDFRenderer(doc);
-        var img = renderer.renderImageWithDPI(pageInformation.number() - 1, DPI, ImageType.GRAY);
-        var imageCtm = getImageCTM(pageInformation, img.getWidth());
-        return findCCBoundingBoxes(img, remove, THRESHOLD, DPI / 72, imageCtm);
-    }
-
-
-    @SneakyThrows
-    private List<Box> findCCBoundingBoxes(BufferedImage image, List<Rectangle2D> remove, int grayScaleTresh, int rescale, AffineTransform imageCTM) {
-
-        var inverseCTM = imageCTM.createInverse();
-
-        var h = image.getHeight();
-        var w = image.getWidth();
-        var pixels = new int[w * h];
-        image.getRaster().getPixels(0, 0, w, h, pixels);
-        remove.stream().map(rect -> inverseCTM.createTransformedShape(rect).getBounds2D()).forEach(box -> {
-            for (int y = (int) Math.floor(box.getMinY() / rescale); y <= (int) Math.min(Math.ceil(box.getMaxY() / rescale), h); y++) {
-                for (int x = (int) Math.floor(box.getMinX() / rescale); x <= (int) Math.min(Math.ceil(box.getMaxX() / rescale), w); x++) {
-                    pixels[w * y + x] = grayScaleTresh;
-                }
-            }
-        });
-
-//        var image2 = createImageFromMatrix(pixels, w, h);
-
-        return findCCBoundingBoxes(pixels, w, h, grayScaleTresh, rescale, imageCTM);
-    }
-
-
-    public static BufferedImage createImageFromMatrix(int[] matrix, int width, int height) {
-
-        BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY);
-
-        byte[] pixelData = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
-
-        for (int y = 0; y < height; y++) {
-            for (int x = 0; x < width; x++) {
-                int index = y * width + x;
-                int pixel = matrix[index]; // Assuming each element in the matrix represents a pixel color
-                pixelData[index] = (byte) pixel;
-            }
-        }
-
-        return image;
-    }
-
-
-    private List<Box> findCCBoundingBoxes(int[] pixels, int w, int h, int pixThreshold, int rescale, AffineTransform imageCTM) {
-
-        DistinctQueue<Integer> pixelsToExplore = new DistinctQueue<>();
-        var boundingBoxes = new ArrayList<Box>();
-
-        for (int y = 0; y < h; y++) {
-            for (int x = 0; x < w; x++) {
-                var pixelIndex = x + y * w;
-                if (pixels[pixelIndex] < pixThreshold) {
-                    var minX = x;
-                    var maxX = x;
-                    var minY = y;
-                    var maxY = y;
-                    pixelsToExplore.enqueue(pixelIndex);
-                    while (!pixelsToExplore.isEmpty()) {
-                        var currentPixel = pixelsToExplore.dequeue();
-                        if (currentPixel > w) {
-                            var lowerPixel = currentPixel - w;
-                            if (pixels[lowerPixel] < pixThreshold) {
-                                pixelsToExplore.enqueue(currentPixel - w);
-                                minY = Math.min(minY, lowerPixel / w);
-                            }
-                        }
-                        if (currentPixel < pixels.length - w) {
-                            var upperPixel = currentPixel + w;
-                            if (pixels[upperPixel] < pixThreshold) {
-                                pixelsToExplore.enqueue(upperPixel);
-                                maxY = Math.max(maxY, upperPixel / w);
-                            }
-                        }
-                        if (currentPixel % w != 0) {
-                            var leftPixel = currentPixel - 1;
-                            if (pixels[leftPixel] < pixThreshold) {
-                                pixelsToExplore.enqueue(leftPixel);
-                                minX = Math.min(minX, leftPixel % w);
-                            }
-                        }
-                        if ((currentPixel + 1) % w != 0) {
-                            var rightPixel = currentPixel + 1;
-                            if (pixels[rightPixel] < pixThreshold) {
-                                pixelsToExplore.enqueue(rightPixel + 1);
-                                maxX = Math.max(maxX, rightPixel % w);
-                            }
-                        }
-                        // Set the current pixel to white so we don't visit it again.
-                        pixels[currentPixel] = pixThreshold;
-                    }
-                    boundingBoxes.add(new Box(minX * rescale, minY * rescale, maxX * rescale, maxY * rescale));
-                }
-            }
-        }
-        return boundingBoxes.stream().filter(box -> box.area() > 0).map(box -> box.transform(imageCTM)).collect(Collectors.toList());
-    }
-
-
-    public AffineTransform getImageCTM(PageInformation pageInformation, int imageWidth) {
-
-        double scalingFactor = calculateScalingFactor(pageInformation, imageWidth);
-        AffineTransform imageToCropBoxScaling = new AffineTransform(scalingFactor, 0, 0, scalingFactor, -pageInformation.minX(), -pageInformation.minY());
-
-        AffineTransform mirrorMatrix = new AffineTransform(1, 0, 0, -1, 0, pageInformation.height());
-
-        AffineTransform rotationMatrix = switch (pageInformation.rotationDegrees()) {
-            case 90 -> new AffineTransform(0, 1, -1, 0, pageInformation.height(), 0);
-            case 180 -> new AffineTransform(-1, 0, 0, -1, pageInformation.width(), pageInformation.height());
-            case 270 -> new AffineTransform(0, -1, 1, 0, pageInformation.width() - pageInformation.height(), pageInformation.height()); // results from 90 + 180 rotations
-            default -> new AffineTransform();
-        };
-
-        // matrix multiplication is performed from right to left, so the order is reversed.
-        // scaling -> mirror -> rotation
-        AffineTransform resultMatrix = new AffineTransform();
-
-        resultMatrix.concatenate(rotationMatrix);
-        resultMatrix.concatenate(mirrorMatrix);
-        resultMatrix.concatenate(imageToCropBoxScaling);
-        return resultMatrix;
-    }
-
-
-    private double calculateScalingFactor(PageInformation pageInformation, int imageWidth) {
-
-        // PDFBox always returns page height and width based on rotation
-        double pageWidth;
-        if (pageInformation.rotationDegrees() == 90 || pageInformation.rotationDegrees() == 270) {
-            pageWidth = pageInformation.height();
-        } else {
-            pageWidth = pageInformation.width();
-        }
-
-        return pageWidth / imageWidth;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/GraphicBBDetector.java

@@ -1,247 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.graphics;
-
-import java.awt.Rectangle;
-import java.awt.color.CMMException;
-import java.awt.geom.GeneralPath;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.pdfbox.contentstream.PDFGraphicsStreamEngine;
-import org.apache.pdfbox.contentstream.PDFStreamEngine;
-import org.apache.pdfbox.contentstream.operator.Operator;
-import org.apache.pdfbox.contentstream.operator.OperatorProcessor;
-import org.apache.pdfbox.cos.COSBase;
-import org.apache.pdfbox.cos.COSName;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
-import org.apache.pdfbox.pdmodel.graphics.image.PDImage;
-
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class GraphicBBDetector extends PDFGraphicsStreamEngine {
-
-    private int clipWindingRule = -1;
-    private final GeneralPath linePath = new GeneralPath();
-    private final List<Rectangle> bounds = new ArrayList<>();
-    private final PDColor emptyPattern = new PDColor(new float[3], null);
-
-
-    public GraphicBBDetector(PDPage page, boolean ignoreWhite) {
-
-        super(page);
-
-        if (!ignoreWhite) {
-            addOperator(new NullOp("d", this));
-            addOperator(new NullOp("k", this));
-            addOperator(new NullOp("K", this));
-            addOperator(new NullOp("g", this));
-            addOperator(new NullOp("G", this));
-            addOperator(new NullOp("CS", this));
-            addOperator(new NullOp("cs", this));
-            addOperator(new NullOp("RG", this));
-            addOperator(new NullOp("rg", this));
-            addOperator(new NullOp("sc", this));
-            addOperator(new NullOp("SC", this));
-            addOperator(new NullOp("scn", this));
-            addOperator(new NullOp("SCN", this));
-        }
-
-        // Ignore text and font ops:
-        addOperator(new NullOp("Tf", this));
-        addOperator(new NullOp("Tj", this));
-        addOperator(new NullOp("TJ", this));
-        addOperator(new NullOp("T*", this));
-        addOperator(new NullOp("'", this));
-        addOperator(new NullOp("\"", this));
-    }
-
-
-    public List<Box> findGraphicBB() throws IOException {
-
-        processPage(getPage());
-        return bounds.stream().map(r -> new Box(r.x, r.y, r.x + r.width, r.y + r.height)).filter(box -> box.area() > 0).collect(Collectors.toList());
-    }
-
-
-    @Override
-    public void appendRectangle(Point2D p0, Point2D p1, Point2D p2, Point2D p3) {
-
-        linePath.moveTo((float) p0.getX(), (float) p0.getY());
-        linePath.lineTo((float) p1.getX(), (float) p1.getY());
-        linePath.lineTo((float) p2.getX(), (float) p2.getY());
-        linePath.lineTo((float) p3.getX(), (float) p3.getY());
-        linePath.closePath();
-    }
-
-
-    @Override
-    public void drawImage(PDImage pdImage) {
-        // Do nothing
-    }
-
-
-    @Override
-    public void clip(int windingRule) {
-
-        clipWindingRule = windingRule;
-    }
-
-
-    @Override
-    public void moveTo(float x, float y) {
-
-        linePath.moveTo(x, y);
-    }
-
-
-    @Override
-    public void lineTo(float x, float y) {
-
-        linePath.lineTo(x, y);
-    }
-
-
-    @Override
-    public void curveTo(float x1, float y1, float x2, float y2, float x3, float y3) {
-
-        linePath.curveTo(x1, y1, x2, y2, x3, y3);
-    }
-
-
-    @Override
-    public Point2D getCurrentPoint() {
-
-        return linePath.getCurrentPoint();
-    }
-
-
-    @Override
-    public void closePath() {
-
-        linePath.closePath();
-    }
-
-
-    @Override
-    public void endPath() {
-
-        if (clipWindingRule != -1) {
-            linePath.setWindingRule(clipWindingRule);
-            getGraphicsState().intersectClippingPath(linePath);
-            clipWindingRule = -1;
-        }
-        linePath.reset();
-    }
-
-
-    @Override
-    public void strokePath() throws IOException {
-
-        addLinePath(true, false);
-        linePath.reset();
-    }
-
-
-    @Override
-    public void fillPath(int windingRule) throws IOException {
-
-        linePath.setWindingRule(windingRule);
-        addLinePath(false, true);
-        linePath.reset();
-    }
-
-
-    @Override
-    public void fillAndStrokePath(int windingRule) throws IOException {
-
-        linePath.setWindingRule(windingRule);
-        addLinePath(true, true);
-        linePath.reset();
-    }
-
-
-    @Override
-    public void shadingFill(COSName shadingName) {
-
-        var newBound = getGraphicsState().getCurrentClippingPath().getBounds();
-        if (newBound.getWidth() > 0 && newBound.getHeight() > 0) {
-            bounds.add(newBound);
-        }
-    }
-
-
-    private void addLinePath(boolean stroke, boolean fill) throws IOException {
-
-        var newBound = getGraphicsState().getCurrentClippingPath().getBounds().intersection(linePath.getBounds());
-        if (newBound.getWidth() > 0 && newBound.getHeight() > 0) {
-
-            if (stroke && !getGraphicsState().getStrokingColor().isPattern() && isBlack(getGraphicsState().getStrokingColor()) || //
-                    !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && isBlack(getGraphicsState().getNonStrokingColor())) {
-                bounds.add(newBound);
-            }
-
-//            var skipWhiteGraphic = ignoreWhite && (!stroke || isWhite(getGraphicsState().getStrokingColor())) && (!fill || isWhite(getGraphicsState().getNonStrokingColor()));
-//            if (!skipWhiteGraphic) {
-//                bounds.add(newBound);
-//            }
-        }
-    }
-
-
-    @SneakyThrows
-    private boolean isBlack(PDColor color) {
-
-        try {
-            return color.toRGB() == 0;
-        } catch (CMMException e) {
-            // see https://github.com/haraldk/TwelveMonkeys/issues/124 or https://issues.apache.org/jira/browse/PDFBOX-3531
-            // This is a quick and dirt hack
-            // Happens for file 216.pdf
-            log.debug(e.getMessage());
-            var result = true;
-            for (var component : color.getComponents()) {
-                result = result && component == 0;
-            }
-            return result;
-        }
-    }
-
-
-    private boolean isWhite(PDColor color) throws IOException {
-
-        return !color.isPattern() && color.toRGB() == 16777215 || color.equals(emptyPattern);
-    }
-
-
-    private final class NullOp extends OperatorProcessor {
-
-        private final String name;
-
-
-        private NullOp(String name, PDFStreamEngine context) {
-
-            super(context);
-            this.name = name;
-        }
-
-
-        @Override
-        public void process(Operator operator, List<COSBase> operands) {
-            // Do nothing.
-        }
-
-
-        @Override
-        public String getName() {
-
-            return name;
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/GraphicExtractorService.java

@@ -1,107 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.graphics;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-
-@Service
-@RequiredArgsConstructor
-public class GraphicExtractorService {
-
-    private final GraphicsClusteringService graphicsClusteringService;
-    private final FindGraphicsRaster findGraphicsRaster;
-
-
-    @SneakyThrows
-    public List<Box> extractPathElementGraphics(PDDocument pdDocument,
-                                                PDPage pdPage,
-                                                int pageNumber,
-                                                CleanRulings cleanRulings,
-                                                List<TextPositionSequence> textPositionSequences,
-                                                List<Cell> emptyTableCells,
-                                                boolean graphicsRaster) {
-
-        var characterBBoxes = getCharacterBBoxes(textPositionSequences);
-        var tableLineBBoxes = getLineBBoxesFromTableCells(emptyTableCells);
-        var underLineBBoxes = getUnderlineBBoxes(cleanRulings, characterBBoxes);
-        var strikeThroughBBoxes = getStrikeThroughBBoxes(cleanRulings, characterBBoxes);
-
-        GraphicBBDetector graphicBBDetector = new GraphicBBDetector(pdPage, true);
-        var graphicBBoxes = graphicBBDetector.findGraphicBB();
-
-        if (graphicsRaster) {
-            // This should only be used if ocr was performed, it is currently in an early stage and needs to be improved.
-            graphicBBoxes.addAll(findGraphicsRaster.findCCBoundingBoxes(pdDocument,
-                    characterBBoxes.stream().map(box -> new Rectangle2D.Double(box.x1 - 2, box.y1 - 2, box.width() + 4, box.height() + 4)).collect(Collectors.toList()),
-                    PageInformation.fromPDPage(pageNumber, pdPage)));
-        }
-
-        var filteredGraphicBBoxes = graphicBBoxes.stream()
-                .filter(box -> !box.intersectsAny(tableLineBBoxes, 4))
-                .filter(box -> !box.intersectsAny(underLineBBoxes, 4))
-                .filter(box -> !box.intersectsAny(strikeThroughBBoxes, 4))
-                .collect(Collectors.toList());
-
-        var clusters = graphicsClusteringService.getClusters(filteredGraphicBBoxes, 14);
-
-        return clusters.stream().filter(box -> box.area() > 500 && box.height() > 50 && box.width() > 50).toList();
-    }
-
-
-    private List<Box> getCharacterBBoxes(List<TextPositionSequence> textPositionSequences) {
-
-        return textPositionSequences.stream()
-                .map(pos -> pos.getTextPositions()
-                        .stream()
-                        .map(tp -> SearchTextWithTextPositionFactory.mapRedTextPositionToInitialUserSpace(tp, pos))
-                        .collect(RectangleTransformations.collectBBox()))
-                .map(Box::new)
-                .collect(Collectors.toList());
-    }
-
-
-    private List<Box> getLineBBoxesFromTableCells(List<Cell> emptyTableCells) {
-
-        List<Box> expandedTableLines = new ArrayList<>();
-
-        emptyTableCells.forEach(cell -> {
-            expandedTableLines.add(new Box(new Rectangle2D.Double(cell.x, cell.y - 1, cell.width, 2)));
-            expandedTableLines.add(new Box(new Rectangle2D.Double(cell.x, cell.y + cell.height - 1, cell.width, 2)));
-            expandedTableLines.add(new Box(new Rectangle2D.Double(cell.x - 1, cell.y, 2, cell.height)));
-            expandedTableLines.add(new Box(new Rectangle2D.Double(cell.x + cell.width - 1, cell.y, 2, cell.height)));
-        });
-
-        return expandedTableLines;
-    }
-
-
-    private List<Box> getUnderlineBBoxes(CleanRulings cleanRulings, List<Box> characterBBoxes) {
-
-        return cleanRulings.getHorizontal()
-                .stream()
-                .map(h -> new Box(h.x1, h.y1, h.x2, h.y2))
-                .filter(box -> box.intersectsAnyAndOver(characterBBoxes, 6))
-                .collect(Collectors.toList());
-    }
-
-
-    private List<Box> getStrikeThroughBBoxes(CleanRulings cleanRulings, List<Box> characterBBoxes) {
-
-        return cleanRulings.getHorizontal().stream().map(h -> new Box(h.x1, h.y1, h.x2, h.y2)).filter(box -> box.intersectsCenter(characterBBoxes, 2)).collect(Collectors.toList());
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/GraphicsClusteringService.java

@@ -1,83 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.graphics;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-@Service
-public class GraphicsClusteringService {
-
-    public List<Box> getClusters(List<Box> boxes, double tol) {
-
-        if (boxes.isEmpty()) {
-            return boxes;
-        } else {
-            // We iteratively pick a Box that intersects at least one other box and replace the
-            // intersecting box with a Box containing them
-            var currentBoxes = boxes;
-            var foundIntersectingBoxes = true;
-            while (foundIntersectingBoxes) {
-                foundIntersectingBoxes = false;
-
-                // The box we are going to check to see if there are any intersecting boxes, followed by
-                // any boxes that we have already check
-                var checked = List.of(currentBoxes.get(0));
-                var unchecked = currentBoxes.subList(1, currentBoxes.size());
-
-                while (!foundIntersectingBoxes && !unchecked.isEmpty()) {
-
-                    List<Box> intersects = new ArrayList<>();
-                    List<Box> nonIntersects = new ArrayList<>();
-
-                    for (Box uncheck : unchecked) {
-                        if (uncheck.intersects(checked.get(0), tol)) {
-                            intersects.add(uncheck);
-                        } else {
-                            nonIntersects.add(uncheck);
-                        }
-                    }
-
-                    if (!intersects.isEmpty()) {
-                        List<Box> combinedIntersecting = new ArrayList<>();
-                        combinedIntersecting.add(checked.get(0));
-                        combinedIntersecting.addAll(intersects);
-                        var newBox = merge(combinedIntersecting);
-
-                        List<Box> newCurrentBoxes = new ArrayList<>();
-                        newCurrentBoxes.add(newBox);
-                        newCurrentBoxes.addAll(checked.subList(1, checked.size()));
-                        newCurrentBoxes.addAll(nonIntersects);
-                        currentBoxes = newCurrentBoxes;
-                        foundIntersectingBoxes = true; // Exit this loop and re-enter the outer loop
-                    } else {
-                        List<Box> newChecked = new ArrayList<>();
-                        newChecked.add(unchecked.get(0));
-                        newChecked.addAll(checked);
-                        checked = newChecked;
-                        unchecked = unchecked.subList(1, unchecked.size());
-                    }
-                }
-            }
-            return currentBoxes;
-        }
-    }
-
-
-    public Box merge(List<Box> boxes) {
-
-        double minX = Double.POSITIVE_INFINITY;
-        double minY = Double.POSITIVE_INFINITY;
-        double maxX = Double.NEGATIVE_INFINITY;
-        double maxY = Double.NEGATIVE_INFINITY;
-
-        for (Box box : boxes) {
-            minX = Math.min(minX, box.x1);
-            minY = Math.min(minY, box.y1);
-            maxX = Math.max(maxX, box.x2);
-            maxY = Math.max(maxY, box.y2);
-        }
-        return new Box(minX, minY, maxX, maxY);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/graphics/PageInformation.java

@@ -1,42 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.graphics;
-
-import java.awt.geom.Rectangle2D;
-
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-
-public record PageInformation(Rectangle2D mediabox, int number, int rotationDegrees) {
-
-    public static PageInformation fromPDPage(int pageNum, PDPage page) {
-
-        PDRectangle mediaBox = page.getMediaBox();
-        return new PageInformation(new Rectangle2D.Double(mediaBox.getLowerLeftX(), mediaBox.getLowerLeftY(), mediaBox.getWidth(), mediaBox.getHeight()),
-                pageNum,
-                page.getRotation());
-    }
-
-
-    public double height() {
-
-        return mediabox.getHeight();
-    }
-
-
-    public double width() {
-
-        return mediabox.getWidth();
-    }
-
-
-    public double minX() {
-
-        return mediabox.getX();
-    }
-
-
-    public double minY() {
-
-        return mediabox.getY();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java

@@ -129,13 +129,11 @@ public class LayoutGridService {
             return;
         }
         for (Page page : table.getPages()) {
-
             Optional<Integer> optionalFirstRowOnPage = table.streamCol(0).filter(tableCell -> tableCell.isOnPage(page.getNumber())).map(TableCell::getRow).findFirst();
             if (optionalFirstRowOnPage.isEmpty()) {
                 continue;
             }
             int firstRowOnPage = optionalFirstRowOnPage.get();
-
             Stream<Double> xStream = switch (page.getRotation()) {
                 case 90 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinX);
                 case 180 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMaxX);
@@ -144,7 +142,6 @@ public class LayoutGridService {
             };
             List<Double> xs = xStream.collect(Collectors.toList());
             xs.remove(0);
-
             Stream<Double> yStream = switch (page.getRotation()) {
                 case 90 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMinY);
                 case 180 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinY);
@@ -154,7 +151,7 @@ public class LayoutGridService {
             List<Double> ys = yStream.collect(Collectors.toList());
             ys.remove(0);
 
-            Rectangle2D tableBBox = table.getBBox().get(page);
+            Rectangle2D tableBBox = table.getBBox().get(table.getFirstPage());
             List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredLines();
             xs.forEach(x -> {
                 Line2D line = new Line2D.Double(new Point2D.Double(x, tableBBox.getMaxY()), new Point2D.Double(x, tableBBox.getMinY()));

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/RectangleTransformations.java

@@ -2,10 +2,9 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 
 import static java.lang.String.format;
 
-import java.awt.geom.Point2D;
+import java.awt.geom.Area;
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -21,8 +20,6 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 
 import lombok.AllArgsConstructor;
 import lombok.NoArgsConstructor;
@@ -43,10 +40,11 @@ public class RectangleTransformations {
 
     public static double calculateIntersectedArea(Rectangle2D r1, Rectangle2D r2) {
 
-        double xOverlap = Math.max(0, Math.min(r1.getMaxX(), r2.getMaxX()) - Math.max(r1.getMinX(), r2.getMinX()));
-        double yOverlap = Math.max(0, Math.min(r1.getMaxY(), r2.getMaxY()) - Math.max(r1.getY(), r2.getY()));
-
-        return xOverlap * yOverlap;
+        Area a1 = new Area(r1);
+        Area a2 = new Area(r2);
+        a1.intersect(a2);
+        Rectangle2D intersection = a1.getBounds2D();
+        return intersection.getWidth() * intersection.getHeight();
     }
 
 
@@ -151,27 +149,7 @@ public class RectangleTransformations {
                 previousRectangle = currentRectangle;
             }
         }
-        return rectangleListsWithGaps.stream()
-                .map(RectangleTransformations::rectangle2DBBox)
-                .toList();
-    }
-
-
-    public static CleanRulings extractRulings(List<? extends Rectangle2D.Float> rectangles) {
-
-        // Underlined or strikethrough are also in rulings but we dont want to split blocks with them so we use cells.
-        List<Ruling> horizontalRulings = new ArrayList<>();
-        List<Ruling> verticalRulings = new ArrayList<>();
-
-        rectangles.forEach(rectangle -> {
-            horizontalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y), new Point2D.Float(rectangle.x + rectangle.width, rectangle.y)));
-            horizontalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y + rectangle.height),
-                                             new Point2D.Float(rectangle.x + rectangle.width, rectangle.y + rectangle.height)));
-            verticalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y), new Point2D.Float(rectangle.x, rectangle.y + rectangle.height)));
-            verticalRulings.add(new Ruling(new Point2D.Float(rectangle.x + rectangle.width, rectangle.y),
-                                           new Point2D.Float(rectangle.x + rectangle.width, rectangle.y + rectangle.height)));
-        });
-        return CleanRulings.builder().vertical(verticalRulings).horizontal(horizontalRulings).build();
+        return rectangleListsWithGaps.stream().map(RectangleTransformations::rectangle2DBBox).toList();
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionOperations.java

@@ -22,9 +22,4 @@ public class TextPositionOperations {
         return sequence;
     }
 
-    public static List<TextPositionSequence> mergeTextPositionSequence(List<TextPageBlock> textBlocks) {
-
-       return textBlocks.stream().flatMap(tb -> tb.getSequences().stream()).collect(Collectors.toList());
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionSequenceComparator.java

@@ -28,13 +28,15 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPosit
  *
  * @author Ben Litchfield
  */
-public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
-
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence>
+{
     @Override
-    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2)
+    {
         // only compare text that is in the same direction
         int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
-        if (cmp1 != 0) {
+        if (cmp1 != 0)
+        {
             return cmp1;
         }
 
@@ -46,19 +48,25 @@ public class TextPositionSequenceComparator implements Comparator<TextPositionSe
         float pos2YBottom = pos2.getMaxYDirAdj();
 
         // note that the coordinates have been adjusted so 0,0 is in upper left
-        float pos1YTop = pos1YBottom - pos1.getTextHeightNoPadding();
-        float pos2YTop = pos2YBottom - pos2.getTextHeightNoPadding();
+        float pos1YTop = pos1YBottom - pos1.getTextHeight();
+        float pos2YTop = pos2YBottom - pos2.getTextHeight();
 
         float yDifference = Math.abs(pos1YBottom - pos2YBottom);
 
         // we will do a simple tolerance comparison
-        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
+        if (yDifference < .1 ||
+                pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
+                pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
+        {
             return Float.compare(x1, x2);
-        } else if (pos1YBottom < pos2YBottom) {
+        }
+        else if (pos1YBottom < pos2YBottom)
+        {
             return -1;
-        } else {
+        }
+        else
+        {
             return 1;
         }
     }
-
 }

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/Application.java

@@ -11,13 +11,14 @@ import org.springframework.context.annotation.Import;
 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
+import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
 import io.micrometer.observation.ObservationRegistry;
 import io.micrometer.observation.aop.ObservedAspect;
 
 @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
-@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class})
+@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {
 

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -1,9 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.server.queue;
 
 import java.util.Arrays;
-import java.util.Locale;
-import java.util.Map;
-import java.util.stream.Collectors;
 
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
@@ -31,7 +28,6 @@ public class MessageHandler {
     private final LayoutParsingPipeline layoutParsingPipeline;
     private final ObjectMapper objectMapper;
     private final RabbitTemplate rabbitTemplate;
-    private final static String X_PIPELINE_PREFIX = "X-PIPE-";
 
 
     @RabbitHandler
@@ -41,7 +37,7 @@ public class MessageHandler {
 
         LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);
 
-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.CLARIFYND) && layoutParsingRequest.researchDocumentStorageId() == null) {
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS) && layoutParsingRequest.researchDocumentStorageId() == null) {
             throw new IllegalArgumentException("ResearchDocumentDataStorageId is null!");
         }
         log.info("Layout parsing request received {}", layoutParsingRequest.identifier());
@@ -50,24 +46,14 @@ public class MessageHandler {
                     layoutParsingRequest.identifier()));
         }
         LayoutParsingFinishedEvent layoutParsingFinishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent, message);
+        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent);
     }
 
 
-    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent, Message message) {
+    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
 
         Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent, m -> {
-            var forwardHeaders = message.getMessageProperties()
-                    .getHeaders()
-                    .entrySet()
-                    .stream()
-                    .filter(e -> e.getKey().toUpperCase(Locale.ROOT).startsWith(X_PIPELINE_PREFIX))
-                    .collect(Collectors.toMap(Map.Entry::getKey,
-                            Map.Entry::getValue));
-            m.getMessageProperties().getHeaders().putAll(forwardHeaders);
-            return m;
-        });
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
     }
 
 }

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java

@@ -0,0 +1,37 @@
+package com.knecon.fforesight.service.layoutparser.server.queue;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+
+import lombok.RequiredArgsConstructor;
+
+@Configuration
+@RequiredArgsConstructor
+public class MessagingConfiguration {
+
+    @Bean
+    public Queue layoutparsingRequestQueue() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    @Bean
+    public Queue layoutparsingResponseQueue() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
+    }
+
+
+    @Bean
+    public Queue layoutparsingDLQ() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -11,7 +11,6 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -49,13 +48,12 @@ public class BdrJsonBuildTest extends AbstractTest {
     @SneakyThrows
     protected Document buildGraph(File file) {
 
-        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.CLARIFYND,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.CLARIFYND,
-                        file,
-                        new ImageServiceResponse(),
-                        new TableServiceResponse(),
-                        new VisualLayoutParsingResponse(),
-                        Map.of("file",file.toString())));
+        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                file,
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
+                file.toString()));
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.server;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -69,11 +68,11 @@ public class HeadlinesGoldStandardIntegrationTest {
     public void testHeadlineDetection() {
 
         List<Metrics> metrics = new ArrayList<>();
-        metrics.add(getMetrics("files/syngenta/CustomerFiles/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf",
+        metrics.add(getMetrics("files/headlineTest/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf",
                 "files/headlineTest/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1)_REDACTION_LOG.json"));
-        metrics.add(getMetrics("files/syngenta/CustomerFiles/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf",
+        metrics.add(getMetrics("files/headlineTest/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf",
                 "files/headlineTest/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23_REDACTION_LOG.json"));
-        metrics.add(getMetrics("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", "files/headlineTest/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
+        metrics.add(getMetrics("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", "files/headlineTest/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
 
         double precision = metrics.stream().mapToDouble(Metrics::getPrecision).average().orElse(1.0);
         double recall = metrics.stream().mapToDouble(Metrics::getRecall).average().orElse(1.0);
@@ -96,13 +95,12 @@ public class HeadlinesGoldStandardIntegrationTest {
         goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
         goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));
 
-        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
-                        pdfFileResource.getFile(),
-                        new ImageServiceResponse(),
-                        new TableServiceResponse(),
-                        new VisualLayoutParsingResponse(),
-                        Map.of("file",filePath)));
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                pdfFileResource.getFile(),
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
+                filePath));
 
         var foundHeadlines = documentGraph.streamAllSubNodes()
                 .map(SemanticNode::getHeadline)

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java

@@ -26,22 +26,9 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
     public void testLayoutParserEndToEnd() {
 
         prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
-        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-        Arrays.stream(finishedEvent.message().split("\n"))
-                .forEach(log::info);
-    }
-
-
-    @Test
-    @SneakyThrows
-    public void testLayoutParserEndToEnd_RED_8747() {
-
-        prepareStorage("files/syngenta/CustomerFiles/SinglePages/Page26_fRR A23317A PI0015600 CEU core part B6 - CZ.pdf");
         LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
         LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-        Arrays.stream(finishedEvent.message().split("\n"))
-                .forEach(log::info);
+        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/SimplifiedTextServiceTest.java

@@ -1,78 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server;
-
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
-
-import io.micrometer.observation.Observation;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class SimplifiedTextServiceTest
-        extends AbstractTest {
-
-    @Autowired
-    private LayoutParsingPipeline layoutParsingPipeline;
-    @Autowired
-    private SimplifiedSectionTextService simplifiedSectionTextService;
-
-
-    @Test
-    @SneakyThrows
-    public void testSearchIndexing() {
-
-        File file = new ClassPathResource("files/headerFooterTest3Pages.pdf").getFile();
-        String footerExample = "Footer to search for ";
-        String headerExample ="Header to search for ";
-        Document document = buildGraph(file);
-        SimplifiedText simplifiedText = simplifiedSectionTextService.toSimplifiedText(document);
-        List<SimplifiedSectionText> sectionTexts = simplifiedText.getSectionTexts();
-        assertThat(sectionTexts.stream().filter(section -> section.getText().equals(footerExample)).collect(Collectors.toList()).size()).isGreaterThan(0);
-        assertThat(sectionTexts.stream().filter(section -> section.getText().equals(headerExample)).collect(Collectors.toList()).size()).isGreaterThan(0);
-
-
-
-    }
-
-    @SneakyThrows
-    protected Document buildGraph(File file) {
-
-        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
-                                                       layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                                                                                         file,
-                                                                                         new ImageServiceResponse(),
-                                                                                         new TableServiceResponse(),
-                                                                                         new VisualLayoutParsingResponse(),
-                                                                                         Map.of("file",file.toString())));
-    }
-
-
-
-}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphJsonWritingTest.java

@@ -4,7 +4,6 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Map;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -48,7 +47,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
     @Disabled
     public void writeJsonForFileTest() {
 
-        var resource = new ClassPathResource("files/syngenta/CustomerFiles/1 Abamectin_prr.pdf");
+        var resource = new ClassPathResource("files/1 Abamectin_prr.pdf");
         writeJsons(resource.getFile().toPath());
     }
 
@@ -56,13 +55,12 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
     @SneakyThrows
     private void writeJsons(Path filename) {
 
-        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
-                        filename.toFile(),
-                        new ImageServiceResponse(),
-                        new TableServiceResponse(),
-                        new VisualLayoutParsingResponse(),
-                        Map.of("file",filename.toFile().toString())));
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                filename.toFile(),
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
+                filename.toFile().toString()));
 
         DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
         ObjectMapper mapper = ObjectMapperFactory.create();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphMappingTest.java

@@ -27,7 +27,7 @@ public class DocumentGraphMappingTest extends BuildDocumentTest {
     @SneakyThrows
     public void testGraphMapping() {
 
-        String filename = "files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf";
+        String filename = "files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf";
 
         Document document = buildGraph(filename);
         DocumentData documentData = DocumentDataMapper.toDocumentData(document);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphVisualizationTest.java

@@ -28,7 +28,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentTest {
 //    @Disabled
     public void visualizeMetolachlor() {
 
-        String filename = "files/syngenta/CustomerFiles/Documine/Flora/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica.pdf";
+        String filename = "files/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (4).pdf";
         visualizePdf(filename);
     }
 
@@ -48,7 +48,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentTest {
     @Disabled
     public void visualizeCraftedDocument() {
 
-        String filename = "files/syngenta/CustomerFiles/1 Abamectin_prr.pdf";
+        String filename = "files/1 Abamectin_prr.pdf";
         visualizePdf(filename);
     }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -2,7 +2,6 @@ package com.knecon.fforesight.service.layoutparser.server.graph;
 
 import java.io.File;
 import java.nio.file.Path;
-import java.util.Map;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -27,15 +26,15 @@ public class ViewerDocumentTest extends BuildDocumentTest {
     @SneakyThrows
     public void testViewerDocument() {
 
-        String fileName = "files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
+        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
         String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
 
         var documentFile = new ClassPathResource(fileName).getFile();
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
 
+        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
         long start = System.currentTimeMillis();
-        Document document = buildGraph(fileName, LayoutParsingType.DOCUMINE);
         layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }
@@ -59,10 +58,10 @@ public class ViewerDocumentTest extends BuildDocumentTest {
                 new ImageServiceResponse(),
                 tableResponse,
                 new VisualLayoutParsingResponse(),
-                Map.of("file", Path.of(fileName).getFileName().toFile().toString()));
+                Path.of(fileName).getFileName().toFile().toString());
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
-        Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE, classificationDocument);
+        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
 
         layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
     }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -25,21 +25,15 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
 import lombok.SneakyThrows;
@@ -67,7 +61,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                 new ImageServiceResponse(),
                 tableServiceResponse,
                 new VisualLayoutParsingResponse(),
-                Map.of("file","document"));
+                "document");
 
         redactManagerClassificationService.classifyDocument(classificationDocument);
 
@@ -87,7 +81,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void tablesToHtmlDebugger() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/T5_Page16_VV-640252.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T5 VV-640252-Page16.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -109,40 +103,6 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     }
 
 
-    @Test
-    public void testHeaderAndFooter() throws IOException {
-
-        String fileName = "files/syngenta/CustomerFiles/54 Fludioxonil - EU AIR3 - Document E1 - Listing of Community and Member States MRLs.pdf";
-        String textToSearch = "Annex to Regulation 283/2013 Annex to Regulation 284/2013";
-        ClassPathResource pdfFileResource = new ClassPathResource(fileName);
-
-        List<PageContents> textPositionPerPage = PageContentExtractor.getSortedPageContents(fileName);
-        var textPositions = textPositionPerPage.stream()
-                .flatMap(t -> t.getSortedTextPositionSequences()
-                        .stream()
-                        .map(TextPositionSequence::toString))
-                .collect(Collectors.joining(" "));
-        assertThat(textPositions.contains(textToSearch)).isFalse();
-
-        ClassificationDocument classificationDocument = buildClassificationDocument(pdfFileResource.getFile());
-
-        assertThat(classificationDocument.getHeaders()
-                           .get(0).getTextBlocks().size()).isEqualTo(3);
-        assertThat(classificationDocument.getHeaders()
-                           .get(0).getTextBlocks()
-                           .get(0).getSequences().size()).isEqualTo(8);
-        assertThat(classificationDocument.getHeaders()
-                           .get(0).getTextBlocks()
-                           .get(0).toString()).isEqualTo(textToSearch);
-
-        Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER, classificationDocument);
-
-        TextBlock leafTextBlock = document.getFirstPage().getHeader().getLeafTextBlock();
-        assertThat(leafTextBlock.getSearchText().contains(textToSearch)).isTrue();
-
-    }
-
-
     @Disabled
     @Test
     public void testScanRotationBorderIsIgnored() throws IOException {
@@ -152,11 +112,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
         var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
 
         // Quality of the table parsing is not good, because the file is rotated at scanning.
@@ -193,7 +149,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testPDFSegmentationWithComplexTable() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Spanning Cells - Page131_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
         assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
@@ -207,130 +163,61 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testTableExtraction() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Merge Table - Page5_26 A8637C - EU AIR3 - LCP Section 10 - Ecotoxicological studies on the plant protection product - Reference list.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(2);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(0)
-                .stream()
-                .map(Collections::singletonList)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
     @Test
     public void testMultiPageMetadataPropagation() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Merge Multi Page Table - Page4_Page5_51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
         assertThat(firstTable.getColCount()).isEqualTo(9);
         assertThat(firstTable.getRowCount()).isEqualTo(5);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
         assertThat(secondTable.getColCount()).isEqualTo(9);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(firstTable.getRowCount() - 1)
-                .stream()
-                .map(Cell::getHeaderCells)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
     @Test
     public void testHeaderCellsForRotatedTable() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Rotated Table Headers - Page4_65 Mesotrione - EU AIR3 - LCA Section 1 Supplement Reference List.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(0)
-                .stream()
-                .map(Collections::singletonList)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
     @Test
     public void testDoc56Page170() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page170_56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -364,7 +251,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testVV931175Page1() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page1_VV-931175.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -405,8 +292,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc27Page6() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Page6_27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -426,7 +312,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Disabled // FIXME Fake Redactions leads to more cells, no solution for this currently
     public void testDocA20622APartB9Page185() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page185_A20622A izRMS (CZ) fRR Part B9.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -439,7 +325,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDocA20622APartB9Page185FixedDoc() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page185_fixed_A20622A izRMS (CZ) fRR Part B9.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185_fixed.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -452,7 +338,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDocA20622APartB7Page123() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page123_A20622A izZRMS (CZ) fRR Part B7.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izZRMS (CZ) fRR Part B7_Page123.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -471,7 +357,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc77Page111() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page11_77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04_Page11.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -487,7 +373,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc95Page532() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page532_95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10_Page532.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -500,7 +386,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc52Page175() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page175_52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -514,7 +400,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc52Page174() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page174_52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -527,8 +413,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc19Page35() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Page35_19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -541,8 +426,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc19Page161() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Page161_19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -557,7 +441,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     public void testDoc47Page30() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Page30_47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance.pdf");
+                "files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -573,7 +457,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     public void testDoc49Page61() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Page61_49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf");
+                "files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -588,8 +472,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc81Page54() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Page54_81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -604,7 +487,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc88Page134() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page134_85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -619,7 +502,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDocThiabendazolePage18() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page18_Thiabendazole DAR Addendum for ED_April_2020.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -636,8 +519,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc15Page18() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Page18_15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -652,7 +534,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     public void testDoc28Page23() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource(
-                "files/syngenta/CustomerFiles/SinglePages/Page23_28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf");
+                "files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -667,7 +549,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc24Page17() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page17_24 - SYN549522 - Acute Oral Toxicity - Rats.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -681,7 +563,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testDoc30Page5() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Page5_30 - Dicamba - Acute Oral Toxicity - Rats.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -744,7 +626,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testT3() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/T3_Page29_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T3 S-Meto_Page29.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -758,7 +640,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testT4() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/T4_Page16_138 IDD0000261736.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T4 138 IDD0000261736_Page16.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -772,7 +654,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void testT5() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/T5_Page16_VV-640252.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T5 VV-640252-Page16.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
@@ -785,19 +667,6 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
     }
 
-    @Test
-    public void testMergedEntities_Page26() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/syngenta/CustomerFiles/SinglePages/Page26_fRR A23317A PI0015600 CEU core part B6 - CZ.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 1);
-
-        validateTable(document, 0, 6, 6, 5, 0);
-
-    }
-
 
     @SneakyThrows
     private void toHtml(ClassificationDocument document, String filename) {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/BodyTextFrameServiceTest.java

@@ -21,7 +21,7 @@ class BodyTextFrameServiceTest extends BuildDocumentTest {
 
         String filename = "files/211.pdf";
         String outputFilename = "/tmp/" + Path.of(filename).getFileName() + "_MAINBODY.pdf";
-        ClassificationDocument document = parseLayout(filename, LayoutParsingType.CLARIFYND);
+        ClassificationDocument document = parseLayout(filename, LayoutParsingType.TAAS);
         PdfDraw.drawRectanglesPerPage(filename,
                 document.getPages().stream().map(page -> List.of(RectangleTransformations.toRectangle2D(page.getBodyTextFrame()))).toList(),
                 outputFilename);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/GapAcrossLinesDetectionServiceTest.java

@@ -26,7 +26,7 @@ class GapAcrossLinesDetectionServiceTest {
     @SneakyThrows
     public void testGapBasedColumnDetection() {
 
-        String filename = "files/basf/CustomerFiles/invisible_tables_test-two-pages_ocred.pdf";
+        String filename = "files/invisible_tables/test-two-pages_ocred.pdf";
         var tmpFileName = "/tmp/" + filename.split("/")[2] + "_COLUMNS.pdf";
         System.out.println("start TextPosition extraction");
         long start = System.currentTimeMillis();
@@ -52,7 +52,7 @@ class GapAcrossLinesDetectionServiceTest {
     @SneakyThrows
     public void testColumnDetection() {
 
-        String filename = "files/basf/CustomerFiles/invisible_tables_test-two-pages_ocred.pdf";
+        String filename = "files/invisible_tables/test-two-pages_ocred.pdf";
         var tmpFileName = "/tmp/" + filename.split("/")[2] + "_COLUMNS.pdf";
         System.out.println("start TextPosition extraction");
         long start = System.currentTimeMillis();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/InvisibleTableDetectionServiceTest.java

@@ -26,7 +26,7 @@ class InvisibleTableDetectionServiceTest {
     @SneakyThrows
     public void detectInvisibleTableTest() {
 
-        String fileName = "files/basf/CustomerFiles/invisible_tables_test-two-pages_ocred.pdf";
+        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
         var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TABLE.pdf").toString();
         List<PageInformation> pageContents = PageContentExtractor.getSortedPageContents(fileName).stream().map(PageInformationService::build).collect(Collectors.toList());
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/MainBodyTextFrameExtractionServiceTest.java

@@ -18,7 +18,7 @@ class MainBodyTextFrameExtractionServiceTest {
     @SneakyThrows
     public void testMainBodyDetection() {
 
-        String fileName = "files/basf/CustomerFiles/invisible_tables_test-two-pages_ocred.pdf";
+        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
         String tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_MAIN_BODY.pdf").toString();
         List<PageContents> sortedTextPositionSequence = PageContentExtractor.getSortedPageContents(fileName);
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/PageContentExtractorTest.java

@@ -20,7 +20,7 @@ class PageContentExtractorTest {
     @SneakyThrows
     public void testTextPositionSequenceExtraction() {
 
-        String fileName = "files/syngenta/CustomerFiles/Documine/Flora/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica.pdf";
+        String fileName = "files/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (4).pdf";
         var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TEXT_POSITION_SEQUENCES.pdf").toString();
 
         List<PageContents> textPositionPerPage = PageContentExtractor.getSortedPageContents(fileName);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/PageInformationServiceTest.java

@@ -20,7 +20,7 @@ class PageInformationServiceTest {
     @SneakyThrows
     public void testGapDetection() {
 
-        String filename = "files/basf/CustomerFiles/invisible_tables_test-two-pages_ocred.pdf";
+        String filename = "files/invisible_tables/test-two-pages_ocred.pdf";
         var tmpFileName = "/tmp/" + filename.split("/")[2] + "_GAPS.pdf";
         System.out.println("start TextPosition extraction");
         long start = System.currentTimeMillis();
@@ -43,7 +43,7 @@ class PageInformationServiceTest {
     @SneakyThrows
     public void testLineDetection() {
 
-        String filename = "files/basf/CustomerFiles/invisible_tables_test-two-pages_ocred.pdf";
+        String filename = "files/invisible_tables/test-two-pages_ocred.pdf";
         var tmpFileName = "/tmp/" + filename.split("/")[2] + "_GAPS.pdf";
         System.out.println("start TextPosition extraction");
         long start = System.currentTimeMillis();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -7,7 +7,6 @@ import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.apache.pdfbox.Loader;
@@ -46,7 +45,7 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
     @SneakyThrows
     public void textRectanglesFromRulingsExtraction() {
 
-        String fileName = "files/syngenta/CustomerFiles/SinglePages/T5_Page16_VV-640252.pdf";
+        String fileName = "files/SinglePages/T5 VV-640252-Page16.pdf";
         String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_CELLS.pdf";
         List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
         RulingCleaningService rulingCleaningService = new RulingCleaningService();
@@ -66,7 +65,7 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
     @SneakyThrows
     public void textRulingExtraction() {
 
-        String fileName = "files/syngenta/CustomerFiles/SinglePages/Page35_19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017.pdf";
+        String fileName = "files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf";
         String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
         List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
         RulingCleaningService rulingCleaningService = new RulingCleaningService();
@@ -100,20 +99,18 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
     @SneakyThrows
     private void writeJsons(Path filename) {
 
-        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
-                        filename.toFile(),
-                        new ImageServiceResponse(),
-                        new TableServiceResponse(),
-                        new VisualLayoutParsingResponse(),
-                        Map.of("file",filename.toFile().toString())));
-        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
-                        filename.toFile(),
-                        new ImageServiceResponse(),
-                        new TableServiceResponse(),
-                        new VisualLayoutParsingResponse(),
-                        Map.of("file",filename.toFile().toString())));
+        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                filename.toFile(),
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
+                filename.toFile().toString()));
+        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                filename.toFile(),
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
+                filename.toFile().toString()));
         DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
         DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
         if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure())) {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;
 
 import java.io.File;
-import java.util.Map;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
@@ -32,7 +31,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
                 layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
                 new TableServiceResponse(),
                 new VisualLayoutParsingResponse(),
-                Map.of("file",filename));
+                filename);
     }
 
 
@@ -46,13 +45,13 @@ public abstract class BuildDocumentTest extends AbstractTest {
     @SneakyThrows
     protected Document buildGraph(String filename, LayoutParsingType layoutParsingType) {
 
-        if (filename.equals("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
+        if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
             prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json");
         } else {
             prepareStorage(filename);
         }
 
-        return DocumentGraphFactory.buildDocumentGraph(layoutParsingType, parseLayout(filename, layoutParsingType));
+        return DocumentGraphFactory.buildDocumentGraph(parseLayout(filename, layoutParsingType));
     }
 
 }