More refactoring

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -18,15 +18,11 @@ public record LayoutParsingRequest(
 
         @Schema(description = "Path to the original PDF file.")//
         @NonNull String originFileStorageId,//
-
-
         @Schema(description = "Optional Path to the table extraction file.")//
         Optional<String> tablesFileStorageId,//
         @Schema(description = "Optional Path to the image classification file.")//
         Optional<String> imagesFileStorageId,//
 
-        @Schema(description = "Optional Path to the the visual layout parsing service file") Optional<String> visualLayoutParsingFileId,//
-
         @Schema(description = "Path where the Document Structure File will be stored.")//
         @NonNull String structureFileStorageId,//
         @Schema(description = "Path where the Research Data File will be stored.")//

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -10,6 +10,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.pdfbox.Loader;
@@ -26,6 +27,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
@@ -33,11 +35,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageB
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
@@ -49,6 +49,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
@@ -88,7 +89,7 @@ public class LayoutParsingPipeline {
     RedactManagerBlockificationService redactManagerBlockificationService;
     LayoutGridService layoutGridService;
     ObservationRegistry observationRegistry;
-    VisualLayoutParsingAdapter visualLayoutParsingAdapter;
+    DocstrumSegmentationService docstrumSegmentationService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -97,33 +98,22 @@ public class LayoutParsingPipeline {
         log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());
 
         File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId())
-                .orElse(originFile);
-
-        VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse();
-        if (layoutParsingRequest.visualLayoutParsingFileId().isPresent()) {
-            visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId().get());
-        }
+        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);
 
         ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId()
-                .isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId()
-                                                                                     .get());
+        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
         }
 
         TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId()
-                .isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId()
-                                                                                     .get());
+        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
         }
 
         ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
-                                                                    originFile,
-                                                                    imageServiceResponse,
-                                                                    tableServiceResponse,
-                                                                    visualLayoutParsingResponse,
+                originFile,
+                imageServiceResponse,
+                tableServiceResponse,
                 layoutParsingRequest.identifier().toString());
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
@@ -156,25 +146,25 @@ public class LayoutParsingPipeline {
                 .numberOfPages(documentGraph.getNumberOfPages())
                 .duration(System.currentTimeMillis() - start)
                 .message(format("""
-                                        Layout parsing has finished in %.02f s.
-                                        identifiers: %s
-                                        %s
-                                        Files have been saved with Ids:
-                                        Structure: %s
-                                        Text: %s
-                                        Positions: %s
-                                        PageData: %s
-                                        Simplified Text: %s
-                                        Viewer Doc: %s""",
-                                ((float) (System.currentTimeMillis() - start)) / 1000,
-                                layoutParsingRequest.identifier(),
-                                buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
-                                layoutParsingRequest.structureFileStorageId(),
-                                layoutParsingRequest.textBlockFileStorageId(),
-                                layoutParsingRequest.positionBlockFileStorageId(),
-                                layoutParsingRequest.pageFileStorageId(),
-                                layoutParsingRequest.simplifiedTextStorageId(),
-                                layoutParsingRequest.viewerDocumentStorageId()))
+                                Layout parsing has finished in %.02f s.
+                                identifiers: %s
+                                %s
+                                Files have been saved with Ids:
+                                Structure: %s
+                                Text: %s
+                                Positions: %s
+                                PageData: %s
+                                Simplified Text: %s
+                                Viewer Doc: %s""",
+                        ((float) (System.currentTimeMillis() - start)) / 1000,
+                        layoutParsingRequest.identifier(),
+                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
+                        layoutParsingRequest.structureFileStorageId(),
+                        layoutParsingRequest.textBlockFileStorageId(),
+                        layoutParsingRequest.positionBlockFileStorageId(),
+                        layoutParsingRequest.pageFileStorageId(),
+                        layoutParsingRequest.simplifiedTextStorageId(),
+                        layoutParsingRequest.viewerDocumentStorageId()))
                 .build();
 
     }
@@ -184,9 +174,9 @@ public class LayoutParsingPipeline {
 
         AtomicReference<Document> documentReference = new AtomicReference<>();
 
-        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry)
-                .contextualName("build-document-graph")
-                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument)));
+        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
+            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
+        });
 
         return documentReference.get();
     }
@@ -195,14 +185,14 @@ public class LayoutParsingPipeline {
     private String buildSemanticNodeCountMessage(int numberOfPages, Map<NodeType, Long> semanticNodeCounts) {
 
         return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
-                             numberOfPages,
-                             semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
-                             semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
-                             semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
-                             semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
-                             semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
-                             semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
-                             semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
+                numberOfPages,
+                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
     }
 
 
@@ -212,17 +202,12 @@ public class LayoutParsingPipeline {
                                               File originFile,
                                               ImageServiceResponse imageServiceResponse,
                                               TableServiceResponse tableServiceResponse,
-                                              VisualLayoutParsingResponse visualLayoutParsingResponse,
                                               String identifier) {
 
         PDDocument originDocument = openDocument(originFile);
         addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
         Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
         Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
-        Map<Integer, List<ClassifiedImage>> signatures = new HashMap<>();
-        if(signatures.size() > 0) {
-            visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse);
-        }
 
         ClassificationDocument classificationDocument = new ClassificationDocument();
         List<ClassificationPage> classificationPages = new ArrayList<>();
@@ -262,11 +247,37 @@ public class LayoutParsingPipeline {
             PDRectangle cropbox = pdPage.getCropBox();
             CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
 
-            ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            };
+            // Docstrum
+            AtomicInteger num = new AtomicInteger(pageNumber);
+            var zones = docstrumSegmentationService.segmentPage(stripper.getTextPositionSequences());
+
+            List<AbstractPageBlock> pageBlocks = new ArrayList<>();
+            AtomicInteger numOnPage = new AtomicInteger(1);
+//            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+            zones.forEach(zone -> {
+
+                List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+                zone.getLines().forEach(line -> {
+                    line.getWords().forEach(word -> {
+                        textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), num.get()));
+                    });
+                });
+
+                var cps = redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
+
+                cps.getTextBlocks().forEach(cp -> {
+                    pageBlocks.add(redactManagerBlockificationService.buildTextBlock(((TextPageBlock) cp).getSequences(), numOnPage.getAndIncrement()));
+                });
+            });
+
+//            ClassificationPage classificationPage = switch (layoutParsingType) {
+//                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
+//                case TAAS -> taasBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
+//                case DOCUMINE -> docuMineBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
+//            };
+
+            ClassificationPage classificationPage = new ClassificationPage(pageBlocks);
+
             classificationPage.setCleanRulings(cleanRulings);
             classificationPage.setRotation(rotation);
             classificationPage.setLandscape(isLandscape);
@@ -283,10 +294,6 @@ public class LayoutParsingPipeline {
                 imageServiceResponseAdapter.findOcr(classificationPage);
             }
 
-            if(signatures.containsKey(pageNumber)) {
-                classificationPage.setImages(signatures.get(pageNumber));
-            }
-
             tableExtractionService.extractTables(cleanRulings, classificationPage);
 
             buildPageStatistics(classificationPage);
@@ -306,9 +313,19 @@ public class LayoutParsingPipeline {
             case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
         }
 
+        List<ClassificationSection> sections = new ArrayList<>();
+        for (var page : classificationPages) {
+            page.getTextBlocks().forEach(block -> {
+                block.setPage(page.getPageNumber());
+                var section = sectionsBuilderService.buildTextBlock(List.of(block), "a");
+                sections.add(section);
+            });
+        }
+        classificationDocument.setSections(sections);
+
         log.info("Building Sections for {}", identifier);
-        sectionsBuilderService.buildSections(classificationDocument);
-        sectionsBuilderService.addImagesToSections(classificationDocument);
+//        sectionsBuilderService.buildSections(classificationDocument);
+//        sectionsBuilderService.addImagesToSections(classificationDocument);
         return classificationDocument;
     }
 
@@ -342,7 +359,9 @@ public class LayoutParsingPipeline {
 
     private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
 
-        document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        if (!classificationPage.isLandscape()) {
+            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        }
         document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
         document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
         document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -10,6 +10,8 @@ import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.Optional;
 
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -20,7 +22,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.Researc
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
@@ -78,14 +79,6 @@ public class LayoutParsingStorageService {
         }
     }
 
-    public VisualLayoutParsingResponse getVisualLayoutParsingFile(String storageId) throws IOException {
-
-        try (InputStream inputStream = getObject(storageId)) {
-            VisualLayoutParsingResponse visualLayoutParsingResponse = objectMapper.readValue(inputStream, VisualLayoutParsingResponse.class);
-            return visualLayoutParsingResponse;
-        }
-    }
-
 
     @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
     public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java

@@ -12,15 +12,16 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRul
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 
 import lombok.Data;
+import lombok.Getter;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
-import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 
 @Data
 @RequiredArgsConstructor
 public class ClassificationPage {
 
     @NonNull
+    @Getter
     private List<AbstractPageBlock> textBlocks;
 
     private List<ClassifiedImage> images = new ArrayList<>();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Cell.java

@@ -76,14 +76,4 @@ public class Cell extends Rectangle {
         return this.getHeight() >= MIN_SIZE && this.getWidth() >= MIN_SIZE;
     }
 
-    public boolean nearlyIntersects(Cell other) {
-
-        if (this.getHeight() <= 0 || other.getHeight() <= 0) {
-            return false;
-        }
-        double x0 = this.getX() + 2;
-        double y0 = this.getY() + 2;
-        return (other.x + other.width > x0 && other.y + other.height > y0 && other.x < x0 + this.getWidth() - 2 && other.y < y0 + this.getHeight() - 2);
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/TablePageBlock.java

@@ -1,12 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;
 
 import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -19,7 +21,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 public class TablePageBlock extends AbstractPageBlock {
 
-    private final TreeMap<CellPosition, Cell> cellTreeMap = new TreeMap<>();
+    private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();
 
     private final int rotation;
     @Getter
@@ -28,14 +30,10 @@ public class TablePageBlock extends AbstractPageBlock {
     private int unrotatedRowCount;
     private int unrotatedColCount;
     private List<List<Cell>> rows;
-    @Getter
-    @Setter
-    private List<Cell> cells;
 
 
     public TablePageBlock(List<Cell> cells, Rectangle area, int rotation) {
 
-        this.cells = cells;
         addCells(cells);
         minX = area.getLeft();
         minY = area.getBottom();
@@ -52,7 +50,6 @@ public class TablePageBlock extends AbstractPageBlock {
         return getColCount() == 0 || getRowCount() == 0;
     }
 
-
     public List<List<Cell>> getRows() {
 
         if (rows == null) {
@@ -83,10 +80,7 @@ public class TablePageBlock extends AbstractPageBlock {
 
     public int getColCount() {
 
-        return getRows().stream()
-                .mapToInt(List::size)
-                .max()
-                .orElse(0);
+        return getRows().stream().mapToInt(List::size).max().orElse(0);
 
     }
 
@@ -126,8 +120,7 @@ public class TablePageBlock extends AbstractPageBlock {
                 List<Cell> cellsToTheTop = new ArrayList<>();
                 for (int i = 0; i < rowIndex; i++) {
                     try {
-                        cellsToTheTop.add(rows.get(i)
-                                                  .get(colIndex));
+                        cellsToTheTop.add(rows.get(i).get(colIndex));
                     } catch (IndexOutOfBoundsException e) {
                         log.debug("No cell {} in row {}, ignoring.", colIndex, rowIndex);
                     }
@@ -142,8 +135,7 @@ public class TablePageBlock extends AbstractPageBlock {
                 if (lastHeaderCell != null) {
                     cell.getHeaderCells().add(lastHeaderCell);
                 }
-                if (!cell.getTextBlocks().isEmpty() && cell.getTextBlocks()
-                        .get(0).getMostPopularWordStyle().equals("bold")) {
+                if (!cell.getTextBlocks().isEmpty() && cell.getTextBlocks().get(0).getMostPopularWordStyle().equals("bold")) {
                     cell.setHeaderCell(true);
                 }
             }
@@ -159,7 +151,7 @@ public class TablePageBlock extends AbstractPageBlock {
             for (int i = 0; i < unrotatedColCount; i++) { // rows
                 List<Cell> lastRow = new ArrayList<>();
                 for (int j = unrotatedRowCount - 1; j >= 0; j--) { // cols
-                    Cell cell = cellTreeMap.get(new CellPosition(j, i));
+                    Cell cell = cells.get(new CellPosition(j, i));
                     if (cell != null) {
                         lastRow.add(cell);
                     }
@@ -170,7 +162,7 @@ public class TablePageBlock extends AbstractPageBlock {
             for (int i = unrotatedColCount - 1; i >= 0; i--) { // rows
                 List<Cell> lastRow = new ArrayList<>();
                 for (int j = 0; j < unrotatedRowCount; j++) { // cols
-                    Cell cell = cellTreeMap.get(new CellPosition(j, i));
+                    Cell cell = cells.get(new CellPosition(j, i));
                     if (cell != null) {
                         lastRow.add(cell);
                     }
@@ -181,7 +173,7 @@ public class TablePageBlock extends AbstractPageBlock {
             for (int i = 0; i < unrotatedRowCount; i++) {
                 List<Cell> lastRow = new ArrayList<>();
                 for (int j = 0; j < unrotatedColCount; j++) {
-                    Cell cell = cellTreeMap.get(new CellPosition(i, j)); // JAVA_8 use getOrDefault()
+                    Cell cell = cells.get(new CellPosition(i, j)); // JAVA_8 use getOrDefault()
                     if (cell != null) {
                         lastRow.add(cell);
                     }
@@ -195,6 +187,17 @@ public class TablePageBlock extends AbstractPageBlock {
     }
 
 
+    private void add(Cell chunk, int row, int col) {
+
+        unrotatedRowCount = Math.max(unrotatedRowCount, row + 1);
+        unrotatedColCount = Math.max(unrotatedColCount, col + 1);
+
+        CellPosition cp = new CellPosition(row, col);
+        cells.put(cp, chunk);
+
+    }
+
+
     private void addCells(List<Cell> cells) {
 
         if (cells.isEmpty()) {
@@ -203,12 +206,11 @@ public class TablePageBlock extends AbstractPageBlock {
 
         cells.removeIf(cell -> cell.getWidth() < 1.1 || cell.getHeight() < 1.1);
 
-        List<List<Cell>> rowsOfCellsMatrix = calculateTableStructure(cells);
+        List<List<Cell>> rowsOfCells = calculateStructure(cells);
 
-        for (int i = 0; i < rowsOfCellsMatrix.size(); i++) {
-            for (int j = 0; j < rowsOfCellsMatrix.get(i).size(); j++) {
-                addCellToRowAndCol(rowsOfCellsMatrix.get(i)
-                                           .get(j), i, j);
+        for (int i = 0; i < rowsOfCells.size(); i++) {
+            for (int j = 0; j < rowsOfCells.get(i).size(); j++) {
+                add(rowsOfCells.get(i).get(j), i, j);
             }
         }
 
@@ -219,36 +221,29 @@ public class TablePageBlock extends AbstractPageBlock {
      * Calculates the structure of the table. For spanning rows and columns multiple cells with the same values will be inserted.
      *
      * @param cells The found cells
-     * @return TablePageBlock Structure as a rows of cells matrix
+     * @return TablePageBlock Structure
      */
-    private List<List<Cell>> calculateTableStructure(List<Cell> cells) {
+    private List<List<Cell>> calculateStructure(List<Cell> cells) {
+
+        List<List<Cell>> matrix = new ArrayList<>();
 
         if (cells.isEmpty()) {
-            return new ArrayList<>();
+            return matrix;
         }
 
         Set<Float> uniqueX = new HashSet<>();
         Set<Float> uniqueY = new HashSet<>();
-        cells.stream()
-                .filter(c -> !c.getTextBlocks().isEmpty() || c.getHeight() > 3 && c.getWidth() > 3)
-                .forEach(c -> {
-                    uniqueX.add(c.getLeft());
-                    uniqueX.add(c.getRight());
-                    uniqueY.add(c.getBottom());
-                    uniqueY.add(c.getTop());
-                });
+        cells.stream().filter(c -> !c.getTextBlocks().isEmpty() || c.getHeight() > 3 && c.getWidth() > 3).forEach(c -> {
+            uniqueX.add(c.getLeft());
+            uniqueX.add(c.getRight());
+            uniqueY.add(c.getBottom());
+            uniqueY.add(c.getTop());
+        });
 
-        var sortedUniqueX = uniqueX.stream()
-                .sorted()
-                .toList();
-        var sortedUniqueY = uniqueY.stream()
-                .sorted()
-                .toList();
-
-        List<List<Cell>> rowsOfCells = new ArrayList<>();
+        var sortedUniqueX = uniqueX.stream().sorted().toList();
+        var sortedUniqueY = uniqueY.stream().sorted().toList();
 
         Float prevY = null;
-
         for (Float y : sortedUniqueY) {
 
             List<Cell> row = new ArrayList<>();
@@ -259,82 +254,43 @@ public class TablePageBlock extends AbstractPageBlock {
                 if (prevY != null && prevX != null) {
                     var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));
 
+                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
+
+                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
                     if (cell.hasMinimumSize()) {
-
-                        cells.stream()
-                                .filter(cell::nearlyIntersects)
-                                .forEach(intersectingCell -> cell.getTextBlocks().addAll(intersectingCell.getTextBlocks()));
-
                         row.add(cell);
                     }
                 }
                 prevX = x;
             }
 
-            // exclude empty rows and rows where all text blocks are empty
-            if (prevY != null && prevX != null && !row.isEmpty() && !row.stream()
-                    .allMatch(cell -> cell.getTextBlocks().isEmpty())) {
-
-                rowsOfCells.add(row);
+            if (prevY != null && prevX != null && !row.isEmpty()) {
+                matrix.add(row);
             }
             prevY = y;
         }
 
-        Collections.reverse(rowsOfCells);
+        Collections.reverse(matrix);
 
-        // now cells are removed which are part of a column without any text blocks
-        // this is done by first computing the inverse matrix which contains call columns of cells
-        // then the column indices that have to be removed are determined
-        List<List<Cell>> columnsOfCells = new ArrayList<>();
-        int maxRowLength = rowsOfCells.stream()
-                .map(List::size)
-                .max(java.util.Comparator.naturalOrder())
-                .orElse(0);
-        for (int i = 0; i < maxRowLength; i++) {
-            columnsOfCells.add(new ArrayList<>());
-        }
-
-        for (List<Cell> row : rowsOfCells) {
-            for (int j = 0; j < row.size(); j++) {
-                columnsOfCells.get(j).add(row.get(j));
-            }
-        }
-
-        List<Integer> columnIndicesToRemove = new ArrayList<>();
-        int columnIndex = 0;
-        for (List<Cell> col : columnsOfCells) {
-            if (col.stream()
-                    .allMatch(cell -> cell.getTextBlocks().isEmpty())) {
-                columnIndicesToRemove.add(columnIndex);
-            }
-            columnIndex++;
-        }
-        columnIndicesToRemove.sort(Collections.reverseOrder());
-
-        // update all rows so that the values of the empty columns get removed
-        var rowsOfCellsBefore = new ArrayList<>(rowsOfCells);
-        rowsOfCells = new ArrayList<>();
-        for (List<Cell> row : rowsOfCellsBefore) {
-            var updatedRow = new ArrayList<>(row);
-            columnIndicesToRemove.forEach(idxToRemove -> updatedRow.remove(updatedRow.get(idxToRemove)));
-            rowsOfCells.add(updatedRow);
-        }
-
-        return rowsOfCells;
+        return matrix;
     }
 
 
-    private void addCellToRowAndCol(Cell cell, int row, int col) {
-
-        unrotatedRowCount = Math.max(unrotatedRowCount, row + 1);
-        unrotatedColCount = Math.max(unrotatedColCount, col + 1);
-
-        CellPosition cp = new CellPosition(row, col);
-        cellTreeMap.put(cp, cell);
 
+    public boolean intersects(Cell cell1, Cell cell2) {
+        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
+            return false;
+        }
+        double x0 = cell1.getX() + 2;
+        double y0 = cell1.getY() + 2;
+        return (cell2.x + cell2.width > x0 &&
+                cell2.y + cell2.height > y0 &&
+                cell2.x < x0 + cell1.getWidth() -2 &&
+                cell2.y < y0 + cell1.getHeight() -2);
     }
 
 
+
     @Override
     public String getText() {
 
@@ -358,7 +314,7 @@ public class TablePageBlock extends AbstractPageBlock {
                             if (!first) {
                                 sb.append("\n");
                             }
-                            sb.append('\"').append(textBlock.getText().replaceAll("\"", "\"")).append('\"');
+                            sb.append('\"').append(textBlock.getText().replaceAll("\"", "\\\"")).append('\"');
                             first = false;
                         }
                     }
@@ -372,6 +328,8 @@ public class TablePageBlock extends AbstractPageBlock {
     }
 
 
+
+
     public String getTextAsHtml() {
 
         StringBuilder sb = new StringBuilder();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -45,6 +45,9 @@ public class RedTextPosition {
     @JsonIgnore
     private String fontName;
 
+    @JsonIgnore
+    private RedTextPosition parent;
+
 
     @SneakyThrows
     public static RedTextPosition fromTextPosition(TextPosition textPosition) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java

@@ -17,6 +17,7 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.Getter;
 import lombok.NoArgsConstructor;
 
 @EqualsAndHashCode(callSuper = true)
@@ -27,6 +28,7 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {
 
     @Builder.Default
+    @Getter
     private List<TextPositionSequence> sequences = new ArrayList<>();
 
     @JsonIgnore
@@ -73,7 +75,7 @@ public class TextPageBlock extends AbstractPageBlock {
 
         return sequences.get(0).getPageWidth();
     }
-    
+
 
     public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {
 
@@ -82,6 +84,7 @@ public class TextPageBlock extends AbstractPageBlock {
         return fromTextPositionSequences(sequences);
     }
 
+
     public static TextPageBlock fromTextPositionSequences(List<TextPositionSequence> wordBlockList) {
 
         TextPageBlock textBlock = null;
@@ -133,7 +136,6 @@ public class TextPageBlock extends AbstractPageBlock {
     }
 
 
-
     /**
      * Returns the minX value in pdf coordinate system.
      * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -55,6 +55,18 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
+    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
+
+        this.textPositions = textPositions;
+        this.page = page;
+        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
+        this.rotation = textPositions.get(0).getRotation();
+        this.pageHeight = textPositions.get(0).getPageHeight();
+        this.pageWidth = textPositions.get(0).getPageWidth();
+        this.isParagraphStart = false;
+    }
+
+
     @Override
     public int length() {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java

@@ -1,83 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingBox;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Service
-@RequiredArgsConstructor
-@Slf4j
-public class VisualLayoutParsingAdapter {
-
-    private static String SIGNATURES = "signature";
-
-    public Map<Integer, List<VisualLayoutParsingResult>> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
-
-        Map<Integer, List<VisualLayoutParsingResult>> tableCells = new HashMap<>();
-        visualLayoutParsingResponse.getData()
-                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertTableCells(tableData.getBoxes())));
-
-        return tableCells;
-
-    }
-
-    public Map<Integer, List<ClassifiedImage>> buildExtractedSignaturesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
-
-        Map<Integer, List<ClassifiedImage>> signatures = new HashMap<>();
-            visualLayoutParsingResponse.getData().forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes())));
-
-            return signatures;
-    }
-
-
-    public List<VisualLayoutParsingResult> convertTableCells(List<VisualLayoutParsingBox> tableObjects) {
-
-        List<VisualLayoutParsingResult> parsedTableCells = new ArrayList<>();
-
-        tableObjects.stream().forEach(t -> {
-            VisualLayoutParsingResult result = new VisualLayoutParsingResult();
-            result.setX0(t.getBox().getX1());
-            result.setX1(t.getBox().getX2());
-            result.setY0(t.getBox().getY1());
-            result.setY1(t.getBox().getY2());
-            result.setWidth(result.getX1() - result.getX0());
-            result.setHeight(result.getY1() - result.getY0());
-            result.setLabel(t.getLabel());
-            parsedTableCells.add(result);
-        });
-
-        return parsedTableCells;
-
-    }
-
-    public List<ClassifiedImage> convertSignatures(int pageNumber, List<VisualLayoutParsingBox> tableObjects) {
-
-        List<ClassifiedImage> signatures = new ArrayList<>();
-
-        tableObjects.stream().forEach(t -> {
-            if(t.getLabel().equals(SIGNATURES)) {
-                ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(),t.getBox().getY1(),t.getBox().getX2() - t.getBox().getX1(),t.getBox().getY2() - t.getBox().getY1()),
-                        ImageType.SIGNATURE,false,pageNumber);
-
-                signatures.add(signature);
-            }
-        });
-
-        return signatures;
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBox.java

@@ -1,18 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class VisualLayoutParsingBox {
-
-    private VisualLayoutParsingBoxValue box;
-    private String label;
-    private float probability;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBoxValue.java

@@ -1,19 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class VisualLayoutParsingBoxValue {
-
-    private float x1;
-    private float y1;
-    private float x2;
-    private float y2;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingData.java

@@ -1,21 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
-
-import java.util.List;
-
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class VisualLayoutParsingData {
-
-    private int page_idx;
-
-    private List<VisualLayoutParsingBox> boxes;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResponse.java

@@ -1,23 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
-
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class VisualLayoutParsingResponse {
-
-    private String dossierId;
-    private String fileId;
-    private String targetFileExtension;
-    private String responseFileExtension;
-    private String X_TENANT_ID;
-    private List<VisualLayoutParsingData> data;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResult.java

@@ -1,22 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class VisualLayoutParsingResult {
-
-    private float x0;
-    private float y0;
-    private float x1;
-    private float y1;
-    private float width;
-    private float height;
-    private String label;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/RulingCleaningService.java

@@ -25,8 +25,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RulingCleaningService {
 
-    private static final float THRESHOLD_Y = 6;
-    private static final float THRESHOLD_X = 2;
+    private static final float THRESHOLD = 6;
 
 
     public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {
@@ -82,7 +81,7 @@ public class RulingCleaningService {
 
         for (Point2D p : points.subList(1, points.size() - 1)) {
             List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD_X) {
+            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
                 groupedPoints.get(groupedPoints.size() - 1).add(p);
             } else {
                 groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -109,7 +108,7 @@ public class RulingCleaningService {
 
         for (Point2D p : points.subList(1, points.size() - 1)) {
             List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD_Y) {
+            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
                 groupedPoints.get(groupedPoints.size() - 1).add(p);
             } else {
                 groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionsBuilderService.java

@@ -240,7 +240,7 @@ public class SectionsBuilderService {
     }
 
 
-    private ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
+    public ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
 
         ClassificationSection section = new ClassificationSection();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/TableExtractionService.java

@@ -5,6 +5,7 @@ import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -24,62 +25,55 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparis
 @Service
 public class TableExtractionService {
 
-    private static final int MAX_TABLE_OUTER_POINT_TOLERANCE = 10;
-    private static final int MAX_TABLE_CONTAINED_CELLS_WITH_TEXT = 1;
-    private static final float SPREADSHEET_AREA_TOLERANCE = 0.001f;
-
-    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (point1, point2) -> {
+    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (arg0, arg1) -> {
 
         int rv = 0;
-        float point1X = DoubleComparisons.round(point1.getX(), 2);
-        float point1Y = DoubleComparisons.round(point1.getY(), 2);
-        float point2X = DoubleComparisons.round(point2.getX(), 2);
-        float point2Y = DoubleComparisons.round(point2.getY(), 2);
+        float arg0X = DoubleComparisons.round(arg0.getX(), 2);
+        float arg0Y = DoubleComparisons.round(arg0.getY(), 2);
+        float arg1X = DoubleComparisons.round(arg1.getX(), 2);
+        float arg1Y = DoubleComparisons.round(arg1.getY(), 2);
 
-        if (point1X > point2X) {
+        if (arg0X > arg1X) {
             rv = 1;
-        } else if (point1X < point2X) {
+        } else if (arg0X < arg1X) {
             rv = -1;
-        } else if (point1Y > point2Y) {
+        } else if (arg0Y > arg1Y) {
             rv = 1;
-        } else if (point1Y < point2Y) {
+        } else if (arg0Y < arg1Y) {
             rv = -1;
         }
         return rv;
     };
-    private static final Comparator<Point2D> Y_FIRST_POINT_COMPARATOR = (point1, point2) -> {
+    private static final Comparator<Point2D> POINT_COMPARATOR = (arg0, arg1) -> {
 
         int rv = 0;
-        float point1X = DoubleComparisons.round(point1.getX(), 2);
-        float point1Y = DoubleComparisons.round(point1.getY(), 2);
-        float point2X = DoubleComparisons.round(point2.getX(), 2);
-        float point2Y = DoubleComparisons.round(point2.getY(), 2);
+        float arg0X = DoubleComparisons.round(arg0.getX(), 2);
+        float arg0Y = DoubleComparisons.round(arg0.getY(), 2);
+        float arg1X = DoubleComparisons.round(arg1.getX(), 2);
+        float arg1Y = DoubleComparisons.round(arg1.getY(), 2);
 
-        if (point1Y > point2Y) {
+        if (arg0Y > arg1Y) {
             rv = 1;
-        } else if (point1Y < point2Y) {
+        } else if (arg0Y < arg1Y) {
             rv = -1;
-        } else if (point1X > point2X) {
+        } else if (arg0X > arg1X) {
             rv = 1;
-        } else if (point1X < point2X) {
+        } else if (arg0X < arg1X) {
             rv = -1;
         }
         return rv;
     };
 
-    private static final Comparator<Cell> CELL_SIZE_COMPARATOR = (cell1, cell2) -> {
 
-        Double cell1Size = cell1.getHeight() * cell1.getWidth();
-        Double cell2Size = cell2.getHeight() * cell2.getWidth();
-        return cell1Size.compareTo(cell2Size);
-    };
+    public boolean contains(Cell cell, double x, double y, double w, double h) {
 
-    private static final Comparator<Rectangle> RECTANGLE_SIZE_COMPARATOR = (rect1, rect2) -> {
-
-        Double rect1Size = rect1.getHeight() * rect1.getWidth();
-        Double rect2Size = rect2.getHeight() * rect2.getWidth();
-        return rect1Size.compareTo(rect2Size);
-    };
+        if (cell.isEmpty() || w <= 0 || h <= 0) {
+            return false;
+        }
+        double x0 = cell.getX();
+        double y0 = cell.getY();
+        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
+    }
 
 
     /**
@@ -95,18 +89,22 @@ public class TableExtractionService {
      * @param cleanRulings The lines used to build the table.
      * @param page         Page object that contains textblocks and statistics.
      */
-
     public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {
 
         List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
-        // sort cells by size (height * width) ascending so that textBlocks are always assigned to the smallest cells that contain them
-        cells.sort(CELL_SIZE_COMPARATOR);
+
+        List<TextPageBlock> toBeRemoved = new ArrayList<>();
 
         for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
             TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
             for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && doesCellContainTextBlock(cell, textBlock)) {
+                if (cell.hasMinimumSize() && contains(cell,
+                        textBlock.getPdfMinX(),
+                        textBlock.getPdfMinY(),
+                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
+                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
                     cell.addTextBlock(textBlock);
+                    toBeRemoved.add(textBlock);
                     break;
                 }
             }
@@ -116,70 +114,39 @@ public class TableExtractionService {
         DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);
 
         List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);
-        // sort spreadsheetAreas by size (height * width) ascending so that cells are placed in the smallest tables first
-        // this way no cell duplication occurs when tables are contained in other tables and only the most inner table contains the cells
-        spreadsheetAreas.sort(RECTANGLE_SIZE_COMPARATOR);
 
         List<TablePageBlock> tables = new ArrayList<>();
         for (Rectangle area : spreadsheetAreas) {
 
-            List<Cell> containedCells = new ArrayList<>();
+            List<Cell> overlappingCells = new ArrayList<>();
             for (Cell c : cells) {
-                if (c.hasMinimumSize() && area.contains(c)) {
-                    containedCells.add(c);
+                if (c.hasMinimumSize() && c.intersects(area)) {
+                    overlappingCells.add(c);
                 }
             }
-
-            var containedCellsWithText = containedCells.stream()
-                    .filter(cell -> !cell.getTextBlocks().isEmpty())
-                    .count();
-
-            // verify if table would contain fewer cells with text than the threshold allows
-            if (containedCellsWithText >= MAX_TABLE_CONTAINED_CELLS_WITH_TEXT) {
-                tables.add(new TablePageBlock(containedCells, area, page.getRotation()));
-                cells.removeAll(containedCells);
-            }
+            tables.add(new TablePageBlock(overlappingCells, area, page.getRotation()));
         }
 
         for (TablePageBlock table : tables) {
             int position = -1;
 
-            for (AbstractPageBlock pageBlock : page.getTextBlocks()) {
-                if (pageBlock instanceof TextPageBlock ? table.containsBlock((TextPageBlock) pageBlock) : table.contains(pageBlock) && position == -1) {
-                    position = page.getTextBlocks().indexOf(pageBlock);
+            Iterator<AbstractPageBlock> itty = page.getTextBlocks().iterator();
+            while (itty.hasNext()) {
+                AbstractPageBlock textBlock = itty.next();
+                if (textBlock instanceof TextPageBlock ? table.containsBlock((TextPageBlock) textBlock) : table.contains(textBlock) && position == -1) {
+                    position = page.getTextBlocks().indexOf(textBlock);
                 }
             }
             if (position != -1) {
                 page.getTextBlocks().add(position, table);
-
-                var toBeRemoved = table.getCells()
-                        .stream()
-                        .map(Cell::getTextBlocks)
-                        .flatMap(List::stream)
-                        .toList();
-                // remove text blocks from the page that were also added with the table (from its contained cells)
-                page.getTextBlocks().removeAll(toBeRemoved);
             }
         }
+
+        page.getTextBlocks().removeAll(toBeRemoved);
     }
 
 
-    private boolean doesCellContainTextBlock(Cell cell, TextPageBlock textBlock) {
-
-        double x = textBlock.getPdfMinX();
-        double y = textBlock.getPdfMinY();
-        double w = textBlock.getPdfMaxX() - textBlock.getPdfMinX();
-        double h = textBlock.getPdfMaxY() - textBlock.getPdfMinY();
-        if (cell.isEmpty() || w <= 0 || h <= 0) {
-            return false;
-        }
-        double x0 = cell.getX();
-        double y0 = cell.getY();
-        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
-    }
-
-
-    private List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
         // Fix for 211.pdf
         for (Ruling r : horizontalRulingLines) {
@@ -193,7 +160,7 @@ public class TableExtractionService {
         List<Cell> cellsFound = new ArrayList<>();
         Map<Point2D, Ruling[]> intersectionPoints = Ruling.findIntersections(horizontalRulingLines, verticalRulingLines);
         List<Point2D> intersectionPointsList = new ArrayList<>(intersectionPoints.keySet());
-        intersectionPointsList.sort(Y_FIRST_POINT_COMPARATOR);
+        intersectionPointsList.sort(POINT_COMPARATOR);
 
         for (int i = 0; i < intersectionPointsList.size(); i++) {
             Point2D topLeft = intersectionPointsList.get(i);
@@ -219,14 +186,13 @@ public class TableExtractionService {
                     continue;
                 }
                 for (Point2D yPoint : yPoints) {
-                    // is there a horizontal edge b/w topLeft and yPoint ?
+                    // is there an horizontal edge b/w topLeft and yPoint ?
                     if (!hv[0].equals(intersectionPoints.get(yPoint)[0])) {
                         continue;
                     }
                     Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY());
-                    if (intersectionPoints.containsKey(btmRight)
-                        && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0])
-                        && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
+                    if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(
+                            intersectionPoints.get(yPoint)[1])) {
                         cellsFound.add(new Cell(topLeft, btmRight));
                         break outer;
                     }
@@ -248,6 +214,7 @@ public class TableExtractionService {
         Set<Point2D> pointSet = new HashSet<>();
         Map<Point2D, Point2D> edgesH = new HashMap<>();
         Map<Point2D, Point2D> edgesV = new HashMap<>();
+        int i = 0;
 
         for (Rectangle cell : cells) {
             for (Point2D pt : cell.getPoints()) {
@@ -264,9 +231,8 @@ public class TableExtractionService {
         pointsSortX.sort(X_FIRST_POINT_COMPARATOR);
         // Y first sort
         List<Point2D> pointsSortY = new ArrayList<>(pointSet);
-        pointsSortY.sort(Y_FIRST_POINT_COMPARATOR);
+        pointsSortY.sort(POINT_COMPARATOR);
 
-        int i = 0;
         while (i < pointSet.size()) {
             float currY = (float) pointsSortY.get(i).getY();
             while (i < pointSet.size() && DoubleComparisons.feq(pointsSortY.get(i).getY(), currY)) {
@@ -291,8 +257,7 @@ public class TableExtractionService {
         Point2D nextVertex;
         while (!edgesH.isEmpty()) {
             ArrayList<PolygonVertex> polygon = new ArrayList<>();
-            Point2D first = edgesH.keySet()
-                    .iterator().next();
+            Point2D first = edgesH.keySet().iterator().next();
             polygon.add(new PolygonVertex(first, Direction.HORIZONTAL));
             edgesH.remove(first);
 
@@ -336,14 +301,7 @@ public class TableExtractionService {
                 bottom = (float) Math.max(bottom, pt.point.getY());
                 right = (float) Math.max(right, pt.point.getX());
             }
-
-            // do not add polygons with too many outer points as they are unlikely to be tables
-            if (poly.size() <= MAX_TABLE_OUTER_POINT_TOLERANCE) {
-                rectangles.add(new Rectangle(top - SPREADSHEET_AREA_TOLERANCE,
-                                             left - SPREADSHEET_AREA_TOLERANCE,
-                                             right - left + 2 * SPREADSHEET_AREA_TOLERANCE,
-                                             bottom - top + 2 * SPREADSHEET_AREA_TOLERANCE));
-            }
+            rectangles.add(new Rectangle(top, left, right - left, bottom - top));
         }
 
         return rectangles;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java

@@ -57,7 +57,7 @@ public class RedactManagerBlockificationService {
             boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
 
-            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
+            if (prev != null && (splitByDir || isSplitByRuling)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList.isEmpty()) {
@@ -167,7 +167,7 @@ public class RedactManagerBlockificationService {
     }
 
 
-    private TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    public TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
         TextPageBlock textBlock = null;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/DocstrumSegmentationService.java

@@ -0,0 +1,153 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.LineBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.NearestNeighbourService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ReadingOrderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.SpacingService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ZoneBuilderService;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class DocstrumSegmentationService {
+
+    private static final double MAX_VERTICAL_MERGE_DIST = 0.5;
+
+    private final NearestNeighbourService nearestNeighbourService;
+    private final SpacingService spacingService;
+    private final LineBuilderService lineBuilderService;
+    private final ZoneBuilderService zoneBuilderService;
+    private final ReadingOrderService readingOrderService;
+
+
+    public List<Zone> segmentPage(List<TextPositionSequence> textPositions) {
+
+        var positions = textPositions.stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
+
+        var characters = positions.stream().map(Character::new).collect(Collectors.toList());
+
+        nearestNeighbourService.findNearestNeighbors(characters);
+
+        var characterSpacing = spacingService.computeCharacterSpacing(characters);
+        var lineSpacing = spacingService.computeLineSpacing(characters);
+
+        var lines = lineBuilderService.buildLines(characters, characterSpacing, lineSpacing);
+
+        var zones = zoneBuilderService.buildZones(lines, characterSpacing, lineSpacing);
+
+        zones = mergeLinesInZones(zones, characterSpacing, Double.NEGATIVE_INFINITY, 0.0, 0.0, lineSpacing * MAX_VERTICAL_MERGE_DIST);
+
+        return readingOrderService.resolve(zones, false);
+    }
+
+//    private List<Zone> mergeZones(List<Zone> zones, double tolerance) {
+//
+//        List<BxBounds> bounds = new ArrayList<BxBounds>(zones.size());
+//        for (List<ComponentLine> zone : zones) {
+//            BxBoundsBuilder builder = new BxBoundsBuilder();
+//            for (ComponentLine line : zone) {
+//                for (Component component : line.getComponents()) {
+//                    builder.expand(component.getChunk().getBounds());
+//                }
+//            }
+//            bounds.add(builder.getBounds());
+//        }
+//
+//        List<List<ComponentLine>> outputZones = new ArrayList<List<ComponentLine>>();
+//        mainFor:
+//        for (int i = 0; i < zones.size(); i++) {
+//            for (int j = 0; j < zones.size(); j++) {
+//                if (i == j || bounds.get(j) == null || bounds.get(i) == null) {
+//                    continue;
+//                }
+//                if (BxModelUtils.contains(bounds.get(j), bounds.get(i), tolerance)) {
+//                    zones.get(j).addAll(zones.get(i));
+//                    bounds.set(i, null);
+//                    continue mainFor;
+//                }
+//            }
+//            outputZones.add(zones.get(i));
+//        }
+//        return outputZones;
+//    }
+
+
+    private List<Zone> mergeLinesInZones(List<Zone> zones,
+                                         double wordSpacing,
+                                         double minHorizontalDistance,
+                                         double maxHorizontalDistance,
+                                         double minVerticalDistance,
+                                         double maxVerticalDistance) {
+
+        List<Zone> outputZones = new ArrayList<>(zones.size());
+        for (Zone zone : zones) {
+            outputZones.add(mergeLinesInZone(zone, wordSpacing, minHorizontalDistance, maxHorizontalDistance, minVerticalDistance, maxVerticalDistance));
+        }
+        return outputZones;
+    }
+
+
+    private Zone mergeLinesInZone(Zone zone,
+                                  double wordSpacing,
+                                  double minHorizontalDistance,
+                                  double maxHorizontalDistance,
+                                  double minVerticalDistance,
+                                  double maxVerticalDistance) {
+
+        DisjointSets<Line> sets = new DisjointSets<>(zone.getLines());
+        for (int i = 0; i < zone.getLines().size(); i++) {
+            Line li = zone.getLines().get(i);
+            for (int j = i + 1; j < zone.getLines().size(); j++) {
+                Line lj = zone.getLines().get(j);
+                double hDist = li.horizontalDistance(lj);
+                double vDist = li.verticalDistance(lj);
+                if (minHorizontalDistance <= hDist && hDist <= maxHorizontalDistance && minVerticalDistance <= vDist && vDist <= maxVerticalDistance) {
+                    sets.union(li, lj);
+                } else if (minVerticalDistance <= vDist && vDist <= maxVerticalDistance && Math.abs(hDist - Math.min(li.getLength(), lj.getLength())) < 0.1) {
+                    boolean componentOverlap = false;
+                    int overlappingCount = 0;
+                    for (Character ci : li.getCharacters()) {
+                        for (Character cj : lj.getCharacters()) {
+                            double dist = ci.overlappingDistance(cj);
+                            if (dist > 2) {
+                                componentOverlap = true;
+                            }
+                            if (dist > 0) {
+                                overlappingCount++;
+                            }
+                        }
+                    }
+                    if (!componentOverlap && overlappingCount <= 2) {
+                        sets.union(li, lj);
+                    }
+                }
+            }
+        }
+        List<Line> outputZone = new ArrayList<>();
+        for (Set<Line> group : sets) {
+            List<Character> components = new ArrayList<>();
+            for (Line line : group) {
+                components.addAll(line.getCharacters());
+            }
+            components.sort(Comparator.comparingDouble(Character::getX));
+
+            outputZone.add(new Line(components, wordSpacing));
+        }
+        return new Zone(outputZone);
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/AngleFilter.java

@@ -0,0 +1,32 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+public class AngleFilter {
+
+    protected double lowerAngle;
+    protected double upperAngle;
+
+
+    public AngleFilter(double lowerAngle, double upperAngle) {
+
+        if (lowerAngle < -Math.PI / 2) {
+            lowerAngle += Math.PI;
+        }
+        if (upperAngle >= Math.PI / 2) {
+            upperAngle -= Math.PI;
+        }
+
+        this.lowerAngle = lowerAngle;
+        this.upperAngle = upperAngle;
+    }
+
+
+    public boolean matches(Neighbor neighbor) {
+
+        if (lowerAngle <= upperAngle) {
+            return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
+        } else {
+            return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/BoundingBox.java

@@ -0,0 +1,48 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+
+import lombok.Data;
+
+@Data
+public abstract class BoundingBox {
+
+    private Rectangle2D bBox;
+
+
+    public double getX() {
+
+        return bBox.getX();
+    }
+
+
+    public double getY() {
+
+        return bBox.getY();
+    }
+
+
+    public double getWidth() {
+
+        return bBox.getWidth();
+    }
+
+
+    public double getHeight() {
+
+        return bBox.getHeight();
+    }
+
+
+    public double getArea() {
+
+        return (bBox.getHeight() * bBox.getWidth());
+    }
+
+
+    public boolean contains(Rectangle2D contained, double tolerance) {
+
+        return bBox.getX() <= contained.getX() + tolerance && bBox.getY() <= contained.getY() + tolerance && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Character.java

@@ -0,0 +1,84 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+
+import lombok.Data;
+
+@Data
+public class Character {
+
+    private final double x;
+    private final double y;
+    private final RedTextPosition textPosition;
+
+    private List<Neighbor> neighbors = new ArrayList<>();
+
+
+    public Character(RedTextPosition chunk) {
+
+        this.x = chunk.getXDirAdj() + chunk.getWidthDirAdj() / 2;
+        this.y = chunk.getYDirAdj() + chunk.getHeightDir() / 2;
+        this.textPosition = chunk;
+    }
+
+
+    public double getHeight() {
+
+        return textPosition.getHeightDir();
+    }
+
+
+    public double distance(Character character) {
+
+        double dx = getX() - character.getX();
+        double dy = getY() - character.getY();
+        return Math.sqrt(dx * dx + dy * dy);
+    }
+
+
+    public double horizontalDistance(Character character) {
+
+        return Math.abs(getX() - character.getX());
+    }
+
+
+    public double verticalDistance(Character character) {
+
+        return Math.abs(getY() - character.getY());
+    }
+
+
+    public double overlappingDistance(Character other) {
+
+        double[] xs = new double[4];
+        double s = Math.sin(-0), c = Math.cos(-0);
+        xs[0] = c * x - s * y;
+        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
+        xs[2] = c * other.x - s * other.y;
+        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
+        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
+        Arrays.sort(xs);
+        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
+    }
+
+
+    public void setNeighbors(List<Neighbor> neighbors) {
+
+        this.neighbors = neighbors;
+    }
+
+
+    public double angle(Character character) {
+
+        if (getX() > character.getX()) {
+            return Math.atan2(getY() - character.getY(), getX() - character.getX());
+        } else {
+            return Math.atan2(character.getY() - getY(), character.getX() - getX());
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/DisjointSets.java

@@ -0,0 +1,194 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.util.AbstractSet;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+public class DisjointSets<E> implements Iterable<Set<E>> {
+
+    private final Map<E, Entry<E>> map = new HashMap<>();
+
+
+    public DisjointSets(Collection<? extends E> collection) {
+
+        for (E element : collection) {
+            map.put(element, new Entry<E>(element));
+        }
+    }
+
+
+    public boolean areTogether(E e1, E e2) {
+
+        return map.get(e1).findRepresentative() == map.get(e2).findRepresentative();
+    }
+
+
+    public void union(E e1, E e2) {
+
+        Entry<E> r1 = map.get(e1).findRepresentative();
+        Entry<E> r2 = map.get(e2).findRepresentative();
+        if (r1 != r2) {
+            if (r1.size <= r2.size) {
+                r2.mergeWith(r1);
+            } else {
+                r1.mergeWith(r2);
+            }
+        }
+    }
+
+
+    @Override
+    public Iterator<Set<E>> iterator() {
+
+        return new Iterator<>() {
+
+            private final Iterator<Entry<E>> iterator = map.values().iterator();
+            private Entry<E> nextRepresentative;
+
+            {
+                findNextRepresentative();
+            }
+
+            @Override
+            public boolean hasNext() {
+
+                return nextRepresentative != null;
+            }
+
+
+            @Override
+            public Set<E> next() {
+
+                if (nextRepresentative == null) {
+                    throw new NoSuchElementException();
+                }
+                Set<E> result = nextRepresentative.asSet();
+                findNextRepresentative();
+                return result;
+            }
+
+
+            private void findNextRepresentative() {
+
+                while (iterator.hasNext()) {
+                    Entry<E> candidate = iterator.next();
+                    if (candidate.isRepresentative()) {
+                        nextRepresentative = candidate;
+                        return;
+                    }
+                }
+                nextRepresentative = null;
+            }
+
+
+            @Override
+            public void remove() {
+
+                throw new UnsupportedOperationException();
+            }
+
+        };
+    }
+
+
+    private static class Entry<E> {
+
+        private int size = 1;
+        private final E value;
+        private Entry<E> parent = this;
+        private Entry<E> next = null;
+        private Entry<E> last = this;
+
+
+        Entry(E value) {
+
+            this.value = value;
+        }
+
+
+        void mergeWith(Entry<E> otherRepresentative) {
+
+            size += otherRepresentative.size;
+            last.next = otherRepresentative;
+            last = otherRepresentative.last;
+            otherRepresentative.parent = this;
+        }
+
+
+        Entry<E> findRepresentative() {
+
+            Entry<E> representative = parent;
+            while (representative.parent != representative) {
+                representative = representative.parent;
+            }
+            for (Entry<E> entry = this; entry != representative; ) {
+                Entry<E> nextEntry = entry.parent;
+                entry.parent = representative;
+                entry = nextEntry;
+            }
+            return representative;
+        }
+
+
+        boolean isRepresentative() {
+
+            return parent == this;
+        }
+
+
+        Set<E> asSet() {
+
+            return new AbstractSet<E>() {
+
+                @Override
+                public Iterator<E> iterator() {
+
+                    return new Iterator<E>() {
+
+                        private Entry<E> nextEntry = findRepresentative();
+
+
+                        @Override
+                        public boolean hasNext() {
+
+                            return nextEntry != null;
+                        }
+
+
+                        @Override
+                        public E next() {
+
+                            if (nextEntry == null) {
+                                throw new NoSuchElementException();
+                            }
+                            E result = nextEntry.value;
+                            nextEntry = nextEntry.next;
+                            return result;
+                        }
+
+
+                        @Override
+                        public void remove() {
+
+                            throw new UnsupportedOperationException();
+                        }
+
+                    };
+                }
+
+
+                @Override
+                public int size() {
+
+                    return findRepresentative().size;
+                }
+            };
+        }
+
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Histogram.java

@@ -0,0 +1,91 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+public class Histogram {
+
+    private static final double EPSILON = 1.0e-6;
+    private final double min;
+    private final double resolution;
+    private double[] frequencies;
+
+
+    public Histogram(double minValue, double maxValue, double resolution) {
+
+        this.min = minValue - EPSILON;
+        double delta = maxValue - minValue + 2 * EPSILON;
+        int size = Math.max(1, (int) Math.round((maxValue - minValue) / resolution));
+        this.resolution = delta / size;
+        this.frequencies = new double[size];
+    }
+
+
+    public void kernelSmooth(double[] kernel) {
+
+        double[] newFrequencies = new double[frequencies.length];
+        int shift = (kernel.length - 1) / 2;
+        for (int i = 0; i < kernel.length; i++) {
+            int jStart = Math.max(0, i - shift);
+            int jEnd = Math.min(frequencies.length, frequencies.length + i - shift);
+            for (int j = jStart; j < jEnd; j++) {
+                newFrequencies[j - i + shift] += kernel[i] * frequencies[j];
+            }
+        }
+        frequencies = newFrequencies;
+    }
+
+
+    public double[] createGaussianKernel(double length, double stdDeviation) {
+
+        int r = (int) Math.round(length / resolution) / 2;
+        stdDeviation /= resolution;
+
+        int size = 2 * r + 1;
+        double[] kernel = new double[size];
+        double sum = 0;
+        double b = 2 * stdDeviation * stdDeviation;
+        double a = 1 / Math.sqrt(Math.PI * b);
+        for (int i = 0; i < size; i++) {
+            kernel[i] = a * Math.exp(-(i - r) * (i - r) / b);
+            sum += kernel[i];
+        }
+        for (int i = 0; i < size; i++) {
+            kernel[i] /= sum;
+        }
+        return kernel;
+    }
+
+
+    public void gaussianSmooth(double windowLength, double stdDeviation) {
+
+        kernelSmooth(createGaussianKernel(windowLength, stdDeviation));
+    }
+
+
+    public void add(double value) {
+
+        frequencies[(int) ((value - min) / resolution)] += 1.0;
+    }
+
+
+    public int getSize() {
+
+        return frequencies.length;
+    }
+
+
+    public double getPeakValue() {
+
+        int peakIndex = 0;
+        for (int i = 1; i < frequencies.length; i++) {
+            if (frequencies[i] > frequencies[peakIndex]) {
+                peakIndex = i;
+            }
+        }
+        int peakEndIndex = peakIndex + 1;
+        final double EPS = 0.0001;
+        while (peakEndIndex < frequencies.length && Math.abs(frequencies[peakEndIndex] - frequencies[peakIndex]) < EPS) {
+            peakEndIndex++;
+        }
+        return ((double) peakIndex + peakEndIndex) / 2 * resolution + min;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Line.java

@@ -0,0 +1,165 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.Data;
+
+@Data
+public class Line extends BoundingBox {
+
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.2;
+
+    private final double x0;
+    private final double y0;
+
+    private final double x1;
+    private final double y1;
+
+    private final double height;
+
+    private final List<Character> characters;
+    private final List<TextPositionSequence> words = new ArrayList<>();
+
+
+    public Line(List<Character> characters, double wordSpacing) {
+
+        this.characters = characters;
+
+        if (characters.size() >= 2) {
+            // linear regression
+            double sx = 0.0, sxx = 0.0, sxy = 0.0, sy = 0.0;
+            for (Character character : characters) {
+                sx += character.getX();
+                sxx += character.getX() * character.getX();
+                sxy += character.getX() * character.getY();
+                sy += character.getY();
+            }
+            double b = (characters.size() * sxy - sx * sy) / (characters.size() * sxx - sx * sx);
+            double a = (sy - b * sx) / characters.size();
+
+            this.x0 = characters.get(0).getX();
+            this.y0 = a + b * this.x0;
+            this.x1 = characters.get(characters.size() - 1).getX();
+            this.y1 = a + b * this.x1;
+        } else {
+            Character character = characters.get(0);
+            double dx = character.getTextPosition().getWidthDirAdj() / 3;
+            double dy = dx * Math.tan(0);
+            this.x0 = character.getX() - dx;
+            this.x1 = character.getX() + dx;
+            this.y0 = character.getY() - dy;
+            this.y1 = character.getY() + dy;
+        }
+        height = computeHeight();
+        computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
+        buildBBox();
+    }
+
+
+    public double getAngle() {
+
+        return Math.atan2(y1 - y0, x1 - x0);
+    }
+
+
+    public double getLength() {
+
+        return Math.sqrt((x0 - x1) * (x0 - x1) + (y0 - y1) * (y0 - y1));
+    }
+
+
+    private double computeHeight() {
+
+        double sum = 0.0;
+        for (Character component : characters) {
+            sum += component.getHeight();
+        }
+        return sum / characters.size();
+    }
+
+
+    public double angularDifference(Line j) {
+
+        double diff = Math.abs(getAngle() - j.getAngle());
+        if (diff <= Math.PI / 2) {
+            return diff;
+        } else {
+            return Math.PI - diff;
+        }
+    }
+
+
+    public double horizontalDistance(Line other) {
+
+        double[] xs = new double[4];
+        xs[0] = x0;
+        xs[1] = x1;
+        xs[2] = other.x0;
+        xs[3] = other.x1;
+        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
+        Arrays.sort(xs);
+        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
+    }
+
+
+    public double verticalDistance(Line other) {
+
+        double ym = (y0 + y1) / 2;
+        double yn = (other.y0 + other.y1) / 2;
+        return Math.abs(ym - yn) / Math.sqrt(1);
+    }
+
+
+    private void computeWords(double wordSpacing) {
+
+        TextPositionSequence word = new TextPositionSequence();
+        Character previous = null;
+        for (Character current : characters) {
+            if (previous != null) {
+                double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
+                if (dist > wordSpacing) {
+                    words.add(word);
+                    word = new TextPositionSequence();
+                }
+            }
+            word.getTextPositions().add(current.getTextPosition());
+            previous = current;
+        }
+        words.add(word);
+    }
+
+
+    private void buildBBox() {
+
+        double minX = Double.POSITIVE_INFINITY;
+        double minY = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+        double maxY = Double.NEGATIVE_INFINITY;
+
+        for (Character character : characters) {
+
+            minX = Math.min(minX, character.getTextPosition().getXDirAdj());
+            minY = Math.min(minY, character.getTextPosition().getYDirAdj());
+            maxX = Math.max(maxX, character.getTextPosition().getXDirAdj() + character.getTextPosition().getWidthDirAdj());
+            maxY = Math.max(maxY, character.getTextPosition().getYDirAdj() + character.getTextPosition().getHeightDir());
+
+        }
+
+        this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
+    }
+
+
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        words.forEach(word -> sb.append(word.toString()).append(" "));
+        return sb.toString().trim();
+    }
+
+}
+

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Neighbor.java

@@ -0,0 +1,36 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import lombok.Getter;
+
+public class Neighbor {
+
+    @Getter
+    private final double distance;
+    @Getter
+    private final double angle;
+    private final Character originCharacter;
+    @Getter
+    private final Character character;
+
+
+    public Neighbor(Character neighbor, Character origin) {
+
+        this.distance = neighbor.distance(origin);
+        this.angle = neighbor.angle(origin);
+        this.character = neighbor;
+        this.originCharacter = origin;
+    }
+
+
+    public double getHorizontalDistance() {
+
+        return character.horizontalDistance(originCharacter);
+    }
+
+
+    public double getVerticalDistance() {
+
+        return character.verticalDistance(originCharacter);
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Zone.java

@@ -0,0 +1,50 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Comparator;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class Zone extends BoundingBox {
+
+    private List<Line> lines;
+
+
+    public Zone(List<Line> lines) {
+
+        lines.sort(Comparator.comparingDouble(Line::getY));
+        this.lines = lines;
+        buildBBox();
+    }
+
+
+    public void buildBBox() {
+
+        double minX = Double.POSITIVE_INFINITY;
+        double minY = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+        double maxY = Double.NEGATIVE_INFINITY;
+
+        for (Line line : lines) {
+
+            minX = Math.min(minX, line.getX());
+            minY = Math.min(minY, line.getY());
+            maxX = Math.max(maxX, line.getX() + line.getWidth());
+            maxY = Math.max(maxY, line.getY() + line.getHeight());
+
+        }
+
+        this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
+    }
+
+
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        lines.forEach(line -> sb.append(line.toString()).append("\n"));
+        return sb.toString().trim();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/LineBuilderService.java

@@ -0,0 +1,50 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+
+@Service
+public class LineBuilderService {
+
+    private static final double CHARACTER_SPACING_DISTANCE_MULTIPLIER = 3.5;
+    private static final double MAX_VERTICAL_CHARACTER_DISTANCE = 0.67;
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+
+
+    public List<Line> buildLines(List<Character> characters, double characterSpacing, double lineSpacing) {
+
+        double maxHorizontalDistance = characterSpacing * CHARACTER_SPACING_DISTANCE_MULTIPLIER;
+        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_CHARACTER_DISTANCE;
+
+        DisjointSets<Character> sets = new DisjointSets<>(characters);
+        AngleFilter filter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
+
+        characters.forEach(character -> {
+            character.getNeighbors().forEach(neighbor -> {
+                double x = neighbor.getHorizontalDistance() / maxHorizontalDistance;
+                double y = neighbor.getVerticalDistance() / maxVerticalDistance;
+                if (filter.matches(neighbor) && Math.pow(x, 2) + Math.pow(y, 2) <= 1) {
+                    sets.union(character, neighbor.getCharacter());
+                }
+            });
+        });
+
+        List<Line> lines = new ArrayList<>();
+        sets.forEach(group -> {
+            List<Character> lineComponents = new ArrayList<>(group);
+            lineComponents.sort(Comparator.comparingDouble(Character::getX));
+            lines.add(new Line(lineComponents, characterSpacing));
+        });
+
+        return lines;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/NearestNeighbourService.java

@@ -0,0 +1,78 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
+
+@Service
+public class NearestNeighbourService {
+
+    private static final int NUMBER_OF_NEIGHBOURS = 8;
+    private static final double STEP = 16.0;
+
+
+    public void findNearestNeighbors(List<Character> characters) {
+
+        if (characters.isEmpty()) {
+            return;
+        }
+
+        characters.sort(Comparator.comparingDouble(Character::getX));
+
+        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
+        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
+            maxNeighborCount = characters.size() - 1;
+        }
+
+        for (int i = 0; i < characters.size(); i++) {
+
+            List<Neighbor> candidates = new ArrayList<>();
+
+            int start = i;
+            int end = i + 1;
+
+            double distance = Double.POSITIVE_INFINITY;
+
+            for (double searchDistance = 0; searchDistance < distance; ) {
+
+                searchDistance += STEP;
+                boolean newCandidatesFound = false;
+
+                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
+                    start--;
+                    candidates.add(new Neighbor(characters.get(start), characters.get(i)));
+                    clearLeastDistant(candidates, maxNeighborCount);
+                    newCandidatesFound = true;
+                }
+
+                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
+                    candidates.add(new Neighbor(characters.get(end), characters.get(i)));
+                    clearLeastDistant(candidates, maxNeighborCount);
+                    end++;
+                    newCandidatesFound = true;
+                }
+
+                if (newCandidatesFound && candidates.size() >= maxNeighborCount) {
+                    distance = candidates.get(maxNeighborCount - 1).getDistance();
+                }
+            }
+            clearLeastDistant(candidates, maxNeighborCount);
+            characters.get(i).setNeighbors(new ArrayList<>(candidates));
+        }
+    }
+
+
+    private void clearLeastDistant(List<Neighbor> candidates, int maxNeighborCount) {
+
+        if (candidates.size() > maxNeighborCount) {
+            candidates.sort(Comparator.comparingDouble(Neighbor::getDistance));
+            candidates.remove(candidates.remove(candidates.size() - 1));
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/ReadingOrderService.java

@@ -0,0 +1,99 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
+
+@Service
+public class ReadingOrderService {
+
+    private static final double THRESHOLD = 1;
+
+
+    public List<Zone> resolve(List<Zone> zones, boolean yxOrder) {
+
+        if (zones.isEmpty() || zones.size() == 1) {
+            return zones;
+        }
+
+        if (yxOrder) {
+            zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+            return zones;
+        }
+
+        return resolveMultiColumnReadingOder(zones);
+    }
+
+
+    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
+
+        // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
+        // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
+
+        double minX = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+
+        for (Zone zone : zones) {
+            if (zone.getX() < minX) {
+                minX = zone.getX();
+            }
+            if (zone.getX() + zone.getWidth() > maxX) {
+                maxX = zone.getX() + zone.getWidth();
+            }
+        }
+
+        double midLineXCoordinate = (minX + maxX) / 2;
+
+        List<Zone> leftOf = new ArrayList<>();
+        List<Zone> rightOf = new ArrayList<>();
+        List<Zone> middle = new ArrayList<>();
+        for (Zone zone : zones) {
+            if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
+                leftOf.add(zone);
+            } else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
+                rightOf.add(zone);
+            } else {
+                middle.add(zone);
+            }
+        }
+
+        leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+
+        rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+
+        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+
+        List<Zone> sortedZones = new ArrayList<>();
+        sortedZones.addAll(leftOf);
+        sortedZones.addAll(rightOf);
+
+        ListIterator<Zone> itty = middle.listIterator();
+
+        while (itty.hasNext()) {
+            Zone current = itty.next();
+            for (int i = 0; i < sortedZones.size(); i++) {
+                if (current.getY() < sortedZones.get(i).getY()) {
+                    sortedZones.add(i, current);
+                    itty.remove();
+                    break;
+                }
+            }
+        }
+
+        sortedZones.addAll(middle);
+
+        return sortedZones;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/SpacingService.java

@@ -0,0 +1,56 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Histogram;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
+
+@Service
+public class SpacingService {
+
+    private static final double SPACING_HISTOGRAM_RESOLUTION = 0.5;
+    private static final double SPACING_HISTOGRAM_SMOOTHING_LENGTH = 2.5;
+    private static final double SPACING_HIST_SMOOTHING_STANDARD_DEVIATION = 0.5;
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+
+
+    public double computeCharacterSpacing(List<Character> components) {
+
+        return computeSpacing(components, 0);
+    }
+
+
+    public double computeLineSpacing(List<Character> components) {
+
+        return computeSpacing(components, Math.PI / 2);
+    }
+
+
+    private double computeSpacing(List<Character> components, double angle) {
+
+        double maxDistance = Double.NEGATIVE_INFINITY;
+
+        for (Character component : components) {
+            for (Neighbor neighbor : component.getNeighbors()) {
+                maxDistance = Math.max(maxDistance, neighbor.getDistance());
+            }
+        }
+        Histogram histogram = new Histogram(0, maxDistance, SPACING_HISTOGRAM_RESOLUTION);
+        AngleFilter filter = new AngleFilter(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
+        for (Character component : components) {
+            for (Neighbor neighbor : component.getNeighbors()) {
+                if (filter.matches(neighbor)) {
+                    histogram.add(neighbor.getDistance());
+                }
+            }
+        }
+
+        histogram.gaussianSmooth(SPACING_HISTOGRAM_SMOOTHING_LENGTH, SPACING_HIST_SMOOTHING_STANDARD_DEVIATION);
+        return histogram.getPeakValue();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/ZoneBuilderService.java

@@ -0,0 +1,90 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+
+@Service
+public class ZoneBuilderService {
+
+    private static final double MIN_HORIZONTAL_DISTANCE_MULTIPLIER = -0.5;
+    private static final double MAX_VERTICAL_DISTANCE_MULTIPLIER = 1.2;
+
+    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -3.0;
+
+    private static final double MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER = 0.5;
+
+    private static final double MIN_LINE_SIZE_SCALE = 0.9;
+
+    private static final double MAX_LINE_SIZE_SCALE = 2.5;
+
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+
+    public static final int MAX_ZONES = 300;
+
+
+    public List<Zone> buildZones(List<Line> lines, double characterSpacing, double lineSpacing) {
+
+        double minHorizontalDistance = characterSpacing * MIN_HORIZONTAL_DISTANCE_MULTIPLIER;
+        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_DISTANCE_MULTIPLIER;
+        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
+        double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;
+
+        DisjointSets<Line> sets = new DisjointSets<>(lines);
+
+        double meanHeight = calculateMeanHeight(lines);
+
+        lines.forEach(outerLine -> //
+                lines.forEach(innerLine -> {
+
+                    double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
+                    scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
+
+                    if (!sets.areTogether(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {
+
+                        double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
+                        double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
+
+                        if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance //
+                                || minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
+                            sets.union(outerLine, innerLine);
+                        }
+                    }
+                }));
+
+        List<Zone> zones = new ArrayList<>();
+        sets.forEach(group -> {
+            zones.add(new Zone(new ArrayList<>(group)));
+        });
+
+        if (zones.size() > MAX_ZONES) {
+            List<Line> oneZoneLines = new ArrayList<>();
+            for (Zone zone : zones) {
+                oneZoneLines.addAll(zone.getLines());
+            }
+            return List.of(new Zone(oneZoneLines));
+        }
+
+        return zones;
+    }
+
+
+    private double calculateMeanHeight(List<Line> lines) {
+
+        double meanHeight = 0.0;
+        double weights = 0.0;
+        for (Line line : lines) {
+            double weight = line.getLength();
+            meanHeight += line.getHeight() * weight;
+            weights += weight;
+        }
+        meanHeight /= weights;
+        return meanHeight;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/utils/DoubleUtils.java

@@ -0,0 +1,18 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils;
+
+public class DoubleUtils {
+
+    public static int compareDouble(double d1, double d2, double precision) {
+
+        if (Double.isNaN(d1) || Double.isNaN(d2)) {
+            return Double.compare(d1, d2);
+        }
+        if (precision == 0) {
+            precision = 1;
+        }
+        long i1 = Math.round(d1 / precision);
+        long i2 = Math.round(d2 / precision);
+        return Long.valueOf(i1).compareTo(i2);
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/PdfDraw.java

@@ -0,0 +1,270 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
+import org.apache.pdfbox.util.Matrix;
+import org.springframework.core.io.ClassPathResource;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class PdfDraw {
+
+    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) throws IOException {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
+                        pageNumber,
+                        rectanglesPerPage.get(pageNumber - 1),
+                        PdfVisualisationUtility.Options.builder().stroke(true).build());
+            }
+            pdDocument.save(out);
+        }
+
+    }
+
+
+    public static void drawRectanglesPerPageNumberedByLine(String filename, List<List<List<Rectangle2D>>> rectanglesPerPage, String tmpFileName) throws IOException {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                var rectanglesOnPage = rectanglesPerPage.get(pageNumber - 1);
+                for (int lineNumber = 0; lineNumber < rectanglesOnPage.size(); lineNumber++) {
+                    var rectanglesInLine = rectanglesOnPage.get(lineNumber);
+                    PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, rectanglesInLine, PdfVisualisationUtility.Options.builder().stroke(true).build());
+                    double y = Math.min(rectanglesInLine.get(0).getMinY(), rectanglesInLine.get(0).getMaxY());
+                    PdfVisualisationUtility.drawText(String.format("%d", lineNumber),
+                            pdDocument,
+                            new Point2D.Double(rectanglesInLine.get(0).getX() - (5 + (5 * countNumberOfDigits(lineNumber))), y + 2),
+                            pageNumber,
+                            PdfVisualisationUtility.Options.builder().stroke(true).build());
+                }
+            }
+            pdDocument.save(out);
+        }
+
+    }
+
+
+    private static int countNumberOfDigits(int num) {
+
+        int final_num = num;
+        if (final_num == 0) {
+            return 1;
+        }
+        int count = 0;
+        for (; final_num != 0; final_num /= 10) {
+            count++;
+        }
+        return count;
+    }
+
+
+    public static void drawDocumentGraph(PDDocument document, Document documentGraph) {
+
+        documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry));
+    }
+
+
+    public static void drawNode(PDDocument document, DocumentTree.Entry entry) {
+
+        Options options = buildStandardOptionsForNodes(entry);
+
+        drawBBoxAndLabelAndNumberOnPage(document, entry, options);
+
+    }
+
+
+    public static void drawTextBlock(PDDocument document, TextBlock textBlock, Options options) {
+
+        textBlock.getAtomicTextBlocks().forEach(atb -> drawAtomicTextBlock(document, atb, options));
+    }
+
+
+    public static void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, Options options) {
+
+        drawRectangle2DList(document, atomicTextBlock.getPage().getNumber(), atomicTextBlock.getPositions().stream().toList(), options);
+
+    }
+
+
+    @SneakyThrows
+    private static void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, Options options, boolean rotate) {
+
+        var pdPage = document.getPage(pageNumber - 1);
+        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
+
+        contentStream.setNonStrokingColor(options.getStrokeColor());
+        contentStream.setLineWidth(options.getStrokeWidth());
+
+        contentStream.beginText();
+        if (rotate) {
+            contentStream.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(15), (float) location.getX(), (float) location.getY()));
+        } else {
+            contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
+        }
+        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
+        contentStream.showText(string);
+        contentStream.endText();
+        contentStream.close();
+    }
+
+
+    @SneakyThrows
+    public static void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, Options options) {
+
+        var pdPage = document.getPage(pageNumber - 1);
+        drawRectangle2DList(document, rectCollection, options, pdPage);
+    }
+
+
+    private static void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, Options options, PDPage pdPage) throws IOException {
+
+        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
+
+        contentStream.setStrokingColor(options.getStrokeColor());
+        contentStream.setNonStrokingColor(options.getFillColor());
+        contentStream.setLineWidth(options.getStrokeWidth());
+
+        for (var r : rectCollection) {
+            contentStream.addRect((float) r.getMinX(), (float) r.getMinY(), (float) r.getWidth(), (float) r.getHeight());
+
+            if (options.isStroke() && options.isFill()) {
+                contentStream.fillAndStroke();
+            } else if (options.isStroke()) {
+                contentStream.stroke();
+            } else if (options.isFill()) {
+                contentStream.fill();
+            }
+        }
+        contentStream.close();
+    }
+
+
+    @SneakyThrows
+    public static void drawRectanglesAndLinesPerPage(String filename, List<List<Rectangle2D>> list, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+//                PdfVisualisationUtility.drawLine2DList(pdDocument,
+//                        pageNumber,
+//                        list.get(pageNumber - 1),
+//                        PdfVisualisationUtility.Options.builder().stroke(true).build());
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
+                        pageNumber,
+                        rectanglesPerPage.get(pageNumber - 1),
+                        PdfVisualisationUtility.Options.builder().stroke(true).build());
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().stroke(true).build());
+            }
+            pdDocument.save(out);
+        }
+    }
+
+
+    @SneakyThrows
+    public static void drawLinesPerPage(String filename, List<List<Ruling>> linesPerPage, String tmpFileName) {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                PdfVisualisationUtility.drawLine2DList(pdDocument,
+                        pageNumber,
+                        linesPerPage.get(pageNumber - 1),
+                        PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
+            }
+            pdDocument.save(out);
+        }
+    }
+
+
+    @Builder
+    @AllArgsConstructor
+    @Getter
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    public static class Options {
+
+        boolean stroke;
+        @Builder.Default
+        Color strokeColor = Color.BLACK;
+        @Builder.Default
+        float strokeWidth = 1f;
+
+        boolean fill;
+        @Builder.Default
+        Color fillColor = Color.BLACK;
+
+    }
+
+
+    private static Options buildStandardOptionsForNodes(DocumentTree.Entry entry) {
+
+        return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
+            case DOCUMENT -> Color.LIGHT_GRAY;
+            case HEADER, FOOTER -> Color.GREEN;
+            case PARAGRAPH -> Color.BLUE;
+            case HEADLINE -> Color.RED;
+            case SECTION -> Color.BLACK;
+            case TABLE -> Color.ORANGE;
+            case TABLE_CELL -> Color.GRAY;
+            case IMAGE -> Color.MAGENTA;
+        }).build();
+    }
+
+
+    private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, Options options) {
+
+        Map<Page, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
+        for (Page page : rectanglesPerPage.keySet()) {
+            Rectangle2D rectangle2D = rectanglesPerPage.get(page);
+            if (entry.getType() == NodeType.SECTION) {
+                rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
+            }
+            drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options);
+            drawText(buildString(entry),
+                    document,
+                    new Point2D.Double(rectangle2D.getMinX(), rectangle2D.getMaxY() + 2),
+                    page.getNumber(),
+                    options,
+                    entry.getType() == NodeType.TABLE_CELL);
+        }
+    }
+
+
+    private static String buildString(DocumentTree.Entry entry) {
+
+        return entry.getNode().getNumberOnPage() + ": " + entry.getTreeId() + ": " + entry.getType();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TableMergingUtility.java

@@ -30,6 +30,8 @@ public class TableMergingUtility {
             if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() && !hasTableHeader(consecutiveTable) && outerBoundaryAlignsX(originalTablePageBlock,
                     consecutiveTable)) {
                 consecutiveTablesWithSameColCountAndHeaders.add(consecutiveTable);
+            } else {
+                break;
             }
         }
         return Stream.concat(Stream.of(originalTablePageBlock), consecutiveTablesWithSameColCountAndHeaders.stream()).toList();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -27,7 +27,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
@@ -51,7 +50,6 @@ public class BdrJsonBuildTest extends AbstractTest {
                 file,
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
                 file.toString()));
     }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -33,7 +33,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 
@@ -99,7 +98,6 @@ public class HeadlinesGoldStandardIntegrationTest {
                 pdfFileResource.getFile(),
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
                 filePath));
 
         var foundHeadlines = documentGraph.streamAllSubNodes()

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java

@@ -25,7 +25,9 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
     @SneakyThrows
     public void testLayoutParserEndToEnd() {
 
-        prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
+        String s = "（";
+        String s1 = "）";
+        prepareStorage("files/Minimal Examples/WrongOrderPage1.pdf");
         LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
         LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
         Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphJsonWritingTest.java

@@ -16,7 +16,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
@@ -59,7 +58,6 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
                 filename.toFile(),
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
                 filename.toFile().toString()));
 
         DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -12,7 +12,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
@@ -26,7 +25,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
     @SneakyThrows
     public void testViewerDocument() {
 
-        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
+        String fileName = "files/Plenarprotokoll 1 (keine Druchsache!) (1).pdf";
         String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
 
         var documentFile = new ClassPathResource(fileName).getFile();
@@ -54,10 +53,10 @@ public class ViewerDocumentTest extends BuildDocumentTest {
         var documentFile = new ClassPathResource(fileName).getFile();
 
         var classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
-                                                                       documentFile,
-                                                                       new ImageServiceResponse(),
-                                                                       tableResponse,
-                                                                       new VisualLayoutParsingResponse(),Path.of(fileName).getFileName().toFile().toString());
+                documentFile,
+                new ImageServiceResponse(),
+                tableResponse,
+                Path.of(fileName).getFileName().toFile().toString());
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
         Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
@@ -66,4 +65,3 @@ public class ViewerDocumentTest extends BuildDocumentTest {
     }
 
 }
-

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -29,9 +29,10 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Im
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
@@ -49,6 +50,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Autowired
     private RedactManagerClassificationService redactManagerClassificationService;
 
+    @Autowired
+    private CvTableParsingAdapter cvTableParsingAdapter;
+
+    @Autowired
+    private ImageServiceResponseAdapter imageServiceResponseAdapter;
+
     @Autowired
     private SectionsBuilderService sectionsBuilderService;
 
@@ -57,11 +64,10 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {
 
         ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                                                                                          originDocument,
-                                                                                          new ImageServiceResponse(),
-                                                                                          tableServiceResponse,
-                                                                                          new VisualLayoutParsingResponse(),
-                                                                                          "document");
+                originDocument,
+                new ImageServiceResponse(),
+                tableServiceResponse,
+                "document");
 
         redactManagerClassificationService.classifyDocument(classificationDocument);
 
@@ -81,11 +87,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @Test
     public void tablesToHtmlDebugger() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T5 VV-640252-Page16.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
 
-        toHtml(document, "/tmp/T5.html");
+        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");
 
     }
 
@@ -103,7 +109,6 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     }
 
 
-    @Disabled
     @Test
     public void testScanRotationBorderIsIgnored() throws IOException {
 
@@ -112,16 +117,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        var tables = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
 
         // Quality of the table parsing is not good, because the file is rotated at scanning.
         // We only asset that the table border is not the page border.
@@ -143,12 +140,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         imageServiceResponse.getData()
                 .forEach(imageMetadata -> images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
                         .add(new ClassifiedImage(new Rectangle2D.Double(imageMetadata.getPosition().getX1(),
-                                                                        imageMetadata.getPosition().getY1(),
-                                                                        imageMetadata.getGeometry().getWidth(),
-                                                                        imageMetadata.getGeometry().getHeight()),
-                                                 ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)),
-                                                 imageMetadata.isAlpha(),
-                                                 imageMetadata.getPosition().getPageNumber())));
+                                imageMetadata.getPosition().getY1(),
+                                imageMetadata.getGeometry().getWidth(),
+                                imageMetadata.getGeometry().getHeight()),
+                                ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)),
+                                imageMetadata.isAlpha(),
+                                imageMetadata.getPosition().getPageNumber())));
 
         System.out.println("object");
     }
@@ -160,22 +157,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock table = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(table.getColCount()).isEqualTo(6);
         assertThat(table.getRowCount()).isEqualTo(13);
-        assertThat(table.getRows()
-                           .stream()
-                           .mapToInt(List::size).sum()).isEqualTo(6 * 13);
+        assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
     }
 
 
@@ -185,37 +171,15 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(2);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(0)
-                .stream()
-                .map(Collections::singletonList)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
@@ -225,37 +189,15 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(9);
         assertThat(firstTable.getRowCount()).isEqualTo(5);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(9);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(firstTable.getRowCount() - 1)
-                .stream()
-                .map(Cell::getHeaderCells)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
@@ -265,41 +207,19 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
 
         ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(0);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
-                .get(0)
-                .stream()
-                .map(Collections::singletonList)
-                .collect(Collectors.toList());
-        assertThat(secondTable.getRows()
-                           .stream()
-                           .allMatch(row -> row.stream()
-                                   .map(Cell::getHeaderCells)
-                                   .toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
-    @Test
+    @Test // Non-sense test
     public void testDoc56Page170() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
@@ -310,8 +230,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
         validateTable(document, 0, 1, 1, 0, 0);
         validateTable(document, 1, 2, 2, 0, 0);
-        validateTable(document, 2, 4, 19, 12, 0);
-        validateTable(document, 3, 2, 12, 0, 0);
+        validateTable(document, 2, 6, 20, 0, 0);
+        validateTable(document, 3, 7, 31, 0, 0);
 
     }
 
@@ -345,30 +265,29 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         validateTable(document, 0, 8, 8, 0, 0);
 
         List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
-                                                                "Author, date",
-                                                                "Study title",
-                                                                "Analytical method Author, date, No.",
-                                                                "Technique, LOQ of the method, validated working range",
-                                                                "Method meets analytical validation criteria",
-                                                                "Remarks (in case validation criteria are not met)",
-                                                                "Acceptability of the method"),
-                                                  Arrays.asList(
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
-                                                  Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
-                                                                "Evans P.G. 2001 TMJ4569B, VV-323245",
-                                                                "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
-                                                                "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
-                                                                "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
-                                                                "Y",
-                                                                "N/A",
-                                                                "Y"));
+                        "Author, date",
+                        "Study title",
+                        "Analytical method Author, date, No.",
+                        "Technique, LOQ of the method, validated working range",
+                        "Method meets analytical validation criteria",
+                        "Remarks (in case validation criteria are not met)",
+                        "Acceptability of the method"),
+                Arrays.asList("Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
+                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
+                        "Evans P.G. 2001 TMJ4569B, VV-323245",
+                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
+                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
+                        "Y",
+                        "N/A",
+                        "Y"));
 
         validateTable(document, 0, values);
 
@@ -660,109 +579,10 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     }
 
 
-    @Test
-    public void testT0() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T0 TableWithMergedCells.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 1);
-
-        validateTable(document, 0, 6, 8, 0, 0);
-    }
-
-
-    @Test
-    public void testT1() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T1 MultipleNestedTable.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 4);
-
-        validateTable(document, 0, 3, 3, 0, 0);
-        validateTable(document, 1, 3, 5, 2, 0);
-        validateTable(document, 2, 3, 3, 1, 0);
-        validateTable(document, 3, 3, 3, 0, 0);
-
-    }
-
-
-    @Test
-    public void testT2() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T2 MultipleTables.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 6);
-
-        validateTable(document, 0, 5, 5, 0, 0);
-        validateTable(document, 1, 5, 6, 0, 0);
-        validateTable(document, 2, 5, 5, 0, 0);
-        validateTable(document, 3, 5, 5, 0, 0);
-        validateTable(document, 4, 5, 5, 0, 0);
-        validateTable(document, 5, 5, 5, 0, 0);
-
-    }
-
-
-    @Test
-    public void testT3() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T3 S-Meto_Page29.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 1);
-
-        validateTable(document, 0, 6, 5, 0, 0);
-
-    }
-
-
-    @Test
-    public void testT4() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T4 138 IDD0000261736_Page16.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 1);
-
-        validateTable(document, 0, 5, 8, 1, 0);
-
-    }
-
-
-    @Test
-    public void testT5() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T5 VV-640252-Page16.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 6);
-        validateTable(document, 0, 1, 1, 0, 0);
-        validateTable(document, 1, 1, 1, 0, 0);
-        validateTable(document, 2, 1, 1, 0, 0);
-        validateTable(document, 3, 1, 1, 0, 0);
-        validateTable(document, 4, 1, 1, 0, 0);
-        validateTable(document, 5, 1, 1, 0, 0);
-
-    }
-
-
     @SneakyThrows
     private void toHtml(ClassificationDocument document, String filename) {
 
-        var tables = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList();
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
         StringBuilder sb = new StringBuilder();
 
         int currentPage = 1;
@@ -783,19 +603,9 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
     private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {
 
-        TablePageBlock table = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(tableIndex);
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
         List<List<Cell>> rows = table.getRows();
-        int emptyCellsFoundFound = rows.stream()
-                .flatMap(List::stream)
-                .toList()
-                .stream()
-                .filter(f -> f.toString().isEmpty())
-                .toList().size();
+        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
 
         for (List<Cell> row : table.getRows()) {
             row.forEach(r -> System.out.println(r.toString()));
@@ -810,20 +620,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
     private void validateTable(ClassificationDocument document, int tableIndex, List<List<String>> values) {
 
-        TablePageBlock table = document.getSections()
-                .stream()
-                .flatMap(paragraph -> paragraph.getTables()
-                        .stream())
-                .toList()
-                .get(tableIndex);
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
         List<List<Cell>> rows = table.getRows();
 
-        List<Cell> rowsFlattened = rows.stream()
-                .flatMap(List::stream)
-                .toList();
-        List<String> valuesFlattened = values.stream()
-                .flatMap(List::stream)
-                .toList();
+        List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();
+        List<String> valuesFlattened = values.stream().flatMap(List::stream).toList();
 
         for (int i = 0; i < valuesFlattened.size(); i++) {
             Cell cell = rowsFlattened.get(i);
@@ -836,11 +637,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
     private void validateTableSize(ClassificationDocument document, int tableSize) {
 
-        assertThat(document.getSections()
-                           .stream()
-                           .flatMap(paragraph -> paragraph.getTables()
-                                   .stream())
-                           .toList().size()).isEqualTo(tableSize);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().size()).isEqualTo(tableSize);
 
     }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -21,14 +21,15 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Ta
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
 
 import lombok.SneakyThrows;
 
@@ -56,6 +57,9 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
     @SneakyThrows
     public void testTableExtraction() {
 
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
+        LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
+
         ClassPathResource resource = new ClassPathResource("files");
         List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
                 .filter(path -> path.getFileName().toString().endsWith(".pdf"))
@@ -63,8 +67,8 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                 .map(Path::toString)
                 .toList();
 
-        for (String pdfFileName : pdfFileNames) {
-            writeJsons(Path.of(pdfFileName));
+        for (int i = 0; i < pdfFileNames.size(); i++) {
+            writeJsons(Path.of(pdfFileNames.get(i)));
         }
     }
 
@@ -76,23 +80,21 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                 filename.toFile(),
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
                 filename.toFile().toString()));
         Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                 filename.toFile(),
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
-                new VisualLayoutParsingResponse(),
                 filename.toFile().toString()));
         DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
         DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
-        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure())) {
-            String tmpFileNameBefore = "/tmp/before." + filename.getFileName().toString();
+        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
+            String tmpFileNameBefore = "C:/Users/YANNIK~1/AppData/Local/Temp/before." + filename.getFileName().toString();
             try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
                 PdfDraw.drawDocumentGraph(pdDocument, documentGraphBefore);
                 pdDocument.save(tmpFileNameBefore);
             }
-            String tmpFileNameAfter = "/tmp/after." + filename.getFileName().toString();
+            String tmpFileNameAfter = "C:/Users/YANNIK~1/AppData/Local/Temp/after." + filename.getFileName().toString();
             try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
                 PdfDraw.drawDocumentGraph(pdDocument, documentGraphAfter);
                 pdDocument.save(tmpFileNameAfter);
@@ -103,9 +105,9 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
 
 
     @SneakyThrows
-    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2) {
+    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2, String pdfName) {
 
-        List<Table> listStructure1 = structure1.streamAllEntries()
+        List listStructure1 = structure1.streamAllEntries()
                 .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
                 .map(DocumentStructure.EntryData::getProperties)
                 .map(properties -> {
@@ -115,7 +117,7 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                 })
                 .toList();
 
-        List<Table> listStructure2 = structure2.streamAllEntries()
+        List listStructure2 = structure2.streamAllEntries()
                 .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
                 .map(DocumentStructure.EntryData::getProperties)
                 .map(properties -> {
@@ -126,8 +128,8 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                 .toList();
 
         for (int i = 0; i < listStructure1.size(); i++) {
-            Table tableNode1 = listStructure1.get(i);
-            Table tableNode2 = listStructure2.get(i);
+            Table tableNode1 = (Table) listStructure1.get(i);
+            Table tableNode2 = (Table) listStructure2.get(i);
             if (tableNode1.getNumberOfRows() != tableNode2.getNumberOfRows() || tableNode1.getNumberOfCols() != tableNode2.getNumberOfCols()) {
                 return false;
             }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -20,7 +20,6 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
-import org.xmlunit.builder.Input;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.storage.commons.service.StorageService;
@@ -52,8 +51,6 @@ public abstract class AbstractTest {
     private RabbitTemplate rabbitTemplate;
 
     protected final static String ORIGIN_FILE_ID = "origin";
-
-    protected final static String VISUAL_LAYOUT_FILE = "visual";
     protected final static String TABLE_FILE_ID = "table";
     protected final static String IMAGE_FILE_ID = "image";
     protected final static String STRUCTURE_FILE_ID = "structure";
@@ -99,7 +96,7 @@ public abstract class AbstractTest {
     @SneakyThrows
     protected LayoutParsingRequest prepareStorage(String file) {
 
-        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json","visual_layout_parsing_response/empty.json");
+        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json");
     }
 
 
@@ -119,7 +116,6 @@ public abstract class AbstractTest {
                 .originFileStorageId(ORIGIN_FILE_ID)
                 .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                 .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
-                .visualLayoutParsingFileId(Optional.of(VISUAL_LAYOUT_FILE))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
                 .positionBlockFileStorageId(POSITION_FILE_ID)
@@ -140,16 +136,6 @@ public abstract class AbstractTest {
         return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream());
     }
 
-    @SneakyThrows
-    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile, String visualLayoutParsingResponseFile) {
-
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
-        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
-        ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
-        ClassPathResource visualLayoutParsingResponseResource = new ClassPathResource(visualLayoutParsingResponseFile);
-
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream(), visualLayoutParsingResponseResource.getInputStream());
-    }
 
     @SneakyThrows
     protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream) {
@@ -161,17 +147,6 @@ public abstract class AbstractTest {
         return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
     }
 
-    @SneakyThrows
-    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream, InputStream visualLayoutParsingResponseFileStream) {
-
-        storageService.storeObject(TenantContext.getTenantId(), IMAGE_FILE_ID, imageInfoStream);
-        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
-        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);
-        storageService.storeObject(TenantContext.getTenantId(),VISUAL_LAYOUT_FILE,visualLayoutParsingResponseFileStream );
-
-        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
-    }
-
 
     @AfterEach
     public void cleanupStorage() {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -10,7 +10,6 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 
 import lombok.SneakyThrows;
@@ -26,7 +25,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
 
         File fileResource = new ClassPathResource(filename).getFile();
         prepareStorage(filename);
-        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), new VisualLayoutParsingResponse(),filename);
+        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), filename);
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/resources/visual_layout_parsing_response/empty.json

@@ -1,8 +0,0 @@
-{
-  "dossierId": "123",
-  "fileId": "123",
-  "targetFileExtension": "ORIGIN.pdf.gz",
-  "responseFileExtension": "EXTRACTED_TABLES.json.gz",
-  "data": [],
-  "X_TENANT_ID": ""
-}