RED-7375 table extractor prototype

2024-02-01 08:23:53 +01:00 · 2024-01-30 11:59:18 +01:00 · 2024-01-17 16:58:53 +01:00 · 2024-01-15 09:43:23 +01:00 · 2023-12-01 13:26:16 +01:00 · 2023-11-29 10:06:05 +01:00
72 changed files with 977 additions and 642 deletions
@@ -10,7 +10,7 @@ deploy:
  script:
    - echo "Building with gradle version ${BUILDVERSION}"
    - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
  artifacts:
    reports:
@@ -20,6 +20,9 @@ public record LayoutParsingRequest(
        @NonNull String originFileStorageId,//
        @Schema(description = "Optional Path to the table extraction file.")//
        Optional<String> tablesFileStorageId,//
+
+        @Schema(description= "Optional Path to the the table parsing service file")
+        Optional<String> tableExtractorFileId,
        @Schema(description = "Optional Path to the image classification file.")//
        Optional<String> imagesFileStorageId,//

@@ -36,9 +39,6 @@ public record LayoutParsingRequest(
        @Schema(description = "Path where the Simplified Text File will be stored.")//
        @NonNull String simplifiedTextStorageId,//
        @Schema(description = "Path where the Viewer Document PDF will be stored.")//
-        @NonNull String viewerDocumentStorageId,//
-        @Deprecated//
-        @Schema(description = "Path where the Section Grid will be stored.")//
-        @NonNull String sectionGridStorageId) {
+        @NonNull String viewerDocumentStorageId) {

 }
@@ -30,12 +30,14 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageB
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.TableExtractorResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
-import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
@@ -64,8 +66,8 @@ public class LayoutParsingPipeline {
    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
    private final CvTableParsingAdapter cvTableParsingAdapter;
    private final LayoutParsingStorageService layoutParsingStorageService;
+    private final TableExtractorResponseAdapter tableExtractorResponseAdapter;
    private final SectionsBuilderService sectionsBuilderService;
-    private final SectionGridCreatorService sectionGridCreatorService;
    private final TaasClassificationService taasClassificationService;
    private final RedactManagerClassificationService redactManagerClassificationService;
    private final DocuMineClassificationService docuMineClassificationService;
@@ -89,22 +91,31 @@ public class LayoutParsingPipeline {
                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
            }

+            TableExtractorResponse tableExtractorResponse = new TableExtractorResponse();
+            if (layoutParsingRequest.tableExtractorFileId().isPresent()) {
+                tableExtractorResponse = layoutParsingStorageService.getExtractedTableFile(layoutParsingRequest.tableExtractorFileId().get());
+            }
+
            TableServiceResponse tableServiceResponse = new TableServiceResponse();
            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
            }

-            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+                    originDocument,
+                    imageServiceResponse,
+                    tableServiceResponse,
+                    tableExtractorResponse);
            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);

            int numberOfPages = originDocument.getNumberOfPages();

-            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));

+            Map<Integer, List<TableExtractorCells>> extractedTableCells = tableExtractorResponseAdapter.buildExtractedTablesPerPage(tableExtractorResponse);
            try (var out = new ByteArrayOutputStream()) {
-                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out, false);
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out,extractedTableCells ,false);
                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
            }

@@ -146,13 +157,13 @@ public class LayoutParsingPipeline {

        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
                numberOfPages,
-                semanticNodeCounts.get(NodeType.SECTION),
-                semanticNodeCounts.get(NodeType.HEADLINE),
-                semanticNodeCounts.get(NodeType.PARAGRAPH),
-                semanticNodeCounts.get(NodeType.TABLE),
-                semanticNodeCounts.get(NodeType.TABLE_CELL),
-                semanticNodeCounts.get(NodeType.HEADER),
-                semanticNodeCounts.get(NodeType.FOOTER));
+                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
    }


@@ -160,11 +171,12 @@ public class LayoutParsingPipeline {
    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
                                              PDDocument originDocument,
                                              ImageServiceResponse imageServiceResponse,
-                                              TableServiceResponse tableServiceResponse) {
+                                              TableServiceResponse tableServiceResponse,
+                                              TableExtractorResponse tableExtractorResponse) {

        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
-
+        Map<Integer, List<TableExtractorCells>> extractedTableCells = tableExtractorResponseAdapter.buildExtractedTablesPerPage(tableExtractorResponse);
        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

@@ -190,10 +202,7 @@ public class LayoutParsingPipeline {
            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);

            PDRectangle cropbox = pdPage.getCropBox();
-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                    stripper.getRulings(),
-                    stripper.getMinCharWidth(),
-                    stripper.getMaxCharHeight());
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());

            ClassificationPage classificationPage = switch (layoutParsingType) {
                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
@@ -216,7 +225,8 @@ public class LayoutParsingPipeline {
                imageServiceResponseAdapter.findOcr(classificationPage);
            }

-            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            tableExtractionService.extractTables(cleanRulings, classificationPage);
+
            buildPageStatistics(classificationPage);
            increaseDocumentStatistics(classificationPage, classificationDocument);

@@ -17,13 +17,13 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

@@ -64,6 +64,16 @@ public class LayoutParsingStorageService {
    }


+    public TableExtractorResponse getExtractedTableFile(String storageId) throws IOException {
+
+        try (InputStream inputStream = getObject(storageId)) {
+            TableExtractorResponse tableExtractorResponse = objectMapper.readValue(inputStream, TableExtractorResponse.class);
+            inputStream.close();
+            return tableExtractorResponse;
+        }
+    }
+
+
    public TableServiceResponse getTablesFile(String storageId) throws IOException {

        try (var tableClassificationStream = getObject(storageId)) {
@@ -84,12 +94,6 @@ public class LayoutParsingStorageService {
    }


-    public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) {
-
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid);
-    }
-
-
    public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.researchDocumentStorageId(), researchDocumentData);
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.ArrayList;
 import java.util.List;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText;

@@ -25,7 +24,6 @@ public class ClassificationDocument {
    private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
    private boolean headlines;

-    private SectionGrid sectionGrid = new SectionGrid();
    private long rulesVersion;

 }
@@ -14,7 +14,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFre
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
-import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;

@Data
@RequiredArgsConstructor
@@ -19,4 +19,5 @@ public class PageContents {
    Rectangle2D cropBox;
    Rectangle2D mediaBox;
    List<Ruling> rulings;
+
 }
@@ -108,11 +108,13 @@ public class Boundary implements Comparable<Boundary> {
        return splitBoundaries;
    }

+
    public IntStream intStream() {

        return IntStream.range(start, end);
    }

+
    public static Boundary merge(Collection<Boundary> boundaries) {

        int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
@@ -105,6 +105,7 @@ public class Document implements GenericSemanticNode {
        return streamAllSubNodes().collect(Collectors.groupingBy(SemanticNode::getType, Collectors.counting()));
    }

+
    @Override
    public String toString() {

@@ -34,7 +34,6 @@ public class Table implements SemanticNode {

    int numberOfRows;
    int numberOfCols;
-
    TextBlock textBlock;

    @Builder.Default
@@ -109,10 +109,7 @@ public class AtomicTextBlock implements TextBlock {
    }


-    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData,
-                                                          DocumentPositionData documentPositionData,
-                                                          SemanticNode parent,
-                                                          Page page) {
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData, DocumentPositionData documentPositionData, SemanticNode parent, Page page) {

        return AtomicTextBlock.builder()
                .id(documentTextData.getId())
@@ -48,6 +48,7 @@ public class TablePageBlock extends AbstractPageBlock {
        return getColCount() == 0 || getRowCount() == 0;
    }

+
    public List<List<Cell>> getRows() {

        if (rows == null) {
@@ -252,7 +253,8 @@ public class TablePageBlock extends AbstractPageBlock {
                if (prevY != null && prevX != null) {
                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));

-                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
+                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
+
                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
                    if (cell.hasMinimumSize()) {
                        row.add(cell);
@@ -273,6 +275,17 @@ public class TablePageBlock extends AbstractPageBlock {
    }


+    public boolean intersects(Cell cell1, Cell cell2) {
+
+        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
+            return false;
+        }
+        double x0 = cell1.getX() + 2;
+        double y0 = cell1.getY() + 2;
+        return (cell2.x + cell2.width > x0 && cell2.y + cell2.height > y0 && cell2.x < x0 + cell1.getWidth() - 2 && cell2.y < y0 + cell1.getHeight() - 2);
+    }
+
+
    @Override
    public String getText() {

@@ -310,8 +323,6 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-
-
    public String getTextAsHtml() {

        StringBuilder sb = new StringBuilder();
@@ -17,7 +17,6 @@ import lombok.SneakyThrows;
@AllArgsConstructor
 public class RedTextPosition {

-    private String textMatrix;
    private float[] position;

    @JsonIgnore
@@ -56,8 +55,6 @@ public class RedTextPosition {

        pos.setFontSizeInPt(textPosition.getFontSizeInPt());

-        pos.setTextMatrix(textPosition.getTextMatrix().toString());
-
        var position = new float[4];

        position[0] = textPosition.getXDirAdj();
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import java.util.ArrayList;
 import java.util.List;
+
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.Getter;
@@ -73,7 +73,7 @@ public class TextPageBlock extends AbstractPageBlock {

        return sequences.get(0).getPageWidth();
    }
-    
+

    public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {

@@ -82,6 +82,7 @@ public class TextPageBlock extends AbstractPageBlock {
        return fromTextPositionSequences(sequences);
    }

+
    public static TextPageBlock fromTextPositionSequences(List<TextPositionSequence> wordBlockList) {

        TextPageBlock textBlock = null;
@@ -133,7 +134,6 @@ public class TextPageBlock extends AbstractPageBlock {
    }


-
    /**
     * Returns the minX value in pdf coordinate system.
     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -224,6 +224,9 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public String getFont() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "none";
+        }
        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
    }

@@ -232,6 +235,9 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public String getFontStyle() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "standard";
+        }
        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);

        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
@@ -243,7 +249,6 @@ public class TextPositionSequence implements CharSequence {
        } else {
            return "standard";
        }
-
    }


@@ -8,6 +8,7 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.PageInfo;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;

@@ -24,24 +25,26 @@ public class CvTableParsingAdapter {
        Map<Integer, List<TableCells>> tableCells = new HashMap<>();
        tableServiceResponse.getData()
                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells())));
+                        .addAll(convertTableCells(tableData.getTableCells(), tableData.getPageInfo())));

        return tableCells;
    }


-    private Collection<TableCells> convertTableCells(List<TableCells> tableCells) {
+    private Collection<TableCells> convertTableCells(List<TableCells> tableCells, PageInfo pageInfo) {

        List<TableCells> cvParsedTableCells = new ArrayList<>();

-        tableCells.forEach(t -> cvParsedTableCells.add(TableCells.builder()
-                .y0(t.getY0())
-                .x1(t.getX1())
-                .y1(t.getY1())
-                .x0(t.getX0())
-                .width(t.getWidth())
-                .height(t.getHeight())
-                .build()));
+        tableCells.stream()
+                .filter(cell -> cell.getWidth() < pageInfo.getWidth() * 0.98 && cell.getHeight() < pageInfo.getHeight() * 0.98)
+                .forEach(t -> cvParsedTableCells.add(TableCells.builder()
+                        .y0(t.getY0())
+                        .x1(t.getX1())
+                        .y1(t.getY1())
+                        .x0(t.getX0())
+                        .width(t.getWidth())
+                        .height(t.getHeight())
+                        .build()));

        return cvParsedTableCells;
    }
@@ -9,10 +9,10 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;

 import lombok.RequiredArgsConstructor;

@@ -20,8 +20,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
 public class ImageServiceResponseAdapter {

-
-    public Map<Integer, List<ClassifiedImage>> buildClassifiedImagesPerPage(ImageServiceResponse imageServiceResponse ) {
+    public Map<Integer, List<ClassifiedImage>> buildClassifiedImagesPerPage(ImageServiceResponse imageServiceResponse) {

        Map<Integer, List<ClassifiedImage>> images = new HashMap<>();
        imageServiceResponse.getData().forEach(imageMetadata -> {
@@ -0,0 +1,74 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTable;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTableData;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableData;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class TableExtractorResponseAdapter {
+
+    public Map<Integer, List<TableExtractorCells>> buildExtractedTablesPerPage(TableExtractorResponse tableExtractorResponse) {
+        Map<Integer, List<TableExtractorCells>> tableCells = new HashMap<>();
+        tableExtractorResponse.getData()
+                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_number(), tableCell -> new ArrayList<>())
+                        .addAll(convertTableCells(tableData.getTables())));
+
+        return tableCells;
+
+    }
+
+    public List<TableExtractorCells> convertTableCells(List<ExtractedTable> tableObjects) {
+
+        List<TableExtractorCells> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            TableExtractorCells tableCells = new TableExtractorCells();
+            tableCells.setX0(t.getTable().getBbox().get(0));
+            tableCells.setX1(t.getTable().getBbox().get(2));
+            tableCells.setY0(t.getTable().getBbox().get(1));
+            tableCells.setY1(t.getTable().getBbox().get(3));
+            tableCells.setWidth(tableCells.getX1()- tableCells.getX0());
+            tableCells.setHeight(tableCells.getY1()- tableCells.getY0());
+            tableCells.setLabel(t.getTable().getLabel());
+            log.info("Parsed table cell {} with label {}",tableCells, tableCells.getLabel());
+            parsedTableCells.add(tableCells);
+            t.getObjects().forEach(o -> {
+                TableExtractorCells objectCell = new TableExtractorCells();
+                objectCell.setX0(t.getTable().getBbox().get(0));
+                objectCell.setX1(t.getTable().getBbox().get(2));
+                objectCell.setY0(t.getTable().getBbox().get(1));
+                objectCell.setY1(t.getTable().getBbox().get(3));
+                objectCell.setWidth(objectCell.getX1()- objectCell.getX0());
+                objectCell.setHeight(objectCell.getY1()- objectCell.getY0());
+                objectCell.setLabel(o.getLabel());
+                log.info("Parsed object cell {} with label {}",objectCell, objectCell.getLabel());
+                parsedTableCells.add(objectCell);
+            });
+        });
+
+        return parsedTableCells;
+
+    }
+
+
+}
@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExtractedTable {
+
+    private boolean rotated;
+    private ExtractedTableData table;
+    private List<ExtractedTableData> objects;
+
+}
@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExtractedTableData {
+
+    private String label;
+    private float score;
+    private List<Float> bbox;
+
+}
@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorCells {
+    private float x0;
+    private float y0;
+    private float x1;
+    private float y1;
+    private float width;
+    private float height;
+    private String label;
+
+}
@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorData {
+
+    private int page_number;
+    private int image;
+    private List<ExtractedTable> tables;
+}
@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorResponse {
+
+    private String dossierId;
+    private String fileId;
+    private String targetFileExtension;
+    private String responseFileExtension;
+    private String X_TENANT_ID;
+    private List<TableExtractorData> data;
+
+}
@@ -25,6 +25,7 @@ public class BodyTextFrameService {
    private static final float RULING_HEIGHT_THRESHOLD = 0.15f; // multiplied with page height. Header/Footer Rulings must be within that border of the page.
    private static final float RULING_WIDTH_THRESHOLD = 0.75f; // multiplied with page width. Header/Footer Rulings must be at least that wide.

+
    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {

        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
@@ -155,8 +156,9 @@ public class BodyTextFrameService {
                        continue;
                    }

-                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || MarkedContentUtils.intersects(textBlock,
+                            page.getMarkedContentBboxPerType(),
+                            MarkedContentUtils.FOOTER)) {
                        continue;
                    }

@@ -22,7 +22,6 @@ public class DividingColumnDetectionService {

    public List<Rectangle2D> detectColumns(PageContents pageContents) {

-
        if (pageContents.getSortedTextPositionSequences().size() < 2) {
            return List.of(pageContents.getCropBox());
        }
@@ -72,11 +72,13 @@ public class GapDetectionService {
        return mirrorY(RectangleTransformations.toRectangle2D(textPosition.getRectangle()));
    }

+
    private static Rectangle2D mirrorY(Rectangle2D rectangle2D) {

        return new Rectangle2D.Double(rectangle2D.getX(), Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY()), rectangle2D.getWidth(), Math.abs(rectangle2D.getHeight()));
    }

+
    private static void addGapToLine(Rectangle2D currentTextPosition, Rectangle2D previousTextPosition, XGapsContext context) {

        context.gapsInCurrentLine.add(new Rectangle2D.Double(previousTextPosition.getMaxX(),
@@ -6,7 +6,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
 import java.util.stream.Stream;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;

@@ -51,7 +50,9 @@ public class GapsAcrossLinesService {
        }

        return columnFactory.outputGaps.stream()
-                .filter(gapAcrossLines -> columnFactory.outputGaps.stream().filter(gapAcrossLines::intersectsX).noneMatch(gapAcrossLines1 -> gapAcrossLines1.lineCount > gapAcrossLines.lineCount))
+                .filter(gapAcrossLines -> columnFactory.outputGaps.stream()
+                        .filter(gapAcrossLines::intersectsX)
+                        .noneMatch(gapAcrossLines1 -> gapAcrossLines1.lineCount > gapAcrossLines.lineCount))
                .filter(gapAcrossLines -> Math.abs(gapAcrossLines.rectangle2D.getMinX() - mainBodyTextFrame.getMinX()) > DISTANCE_TO_BORDER_THRESHOLD)
                .filter(gapAcrossLines -> Math.abs(gapAcrossLines.rectangle2D.getMaxX() - mainBodyTextFrame.getMaxX()) > DISTANCE_TO_BORDER_THRESHOLD)
                .map(GapAcrossLines::getRectangle2D)
@@ -6,8 +6,8 @@ import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -16,8 +16,7 @@ public class MainBodyTextFrameExtractionService {

    public Rectangle2D calculateMainBodyTextFrame(LineInformation lineInformation) {

-        Rectangle2D mainBodyTextFrame = lineInformation.getLineBBox().stream()
-                .collect(RectangleTransformations.collectBBox());
+        Rectangle2D mainBodyTextFrame = lineInformation.getLineBBox().stream().collect(RectangleTransformations.collectBBox());

        return RectangleTransformations.pad(mainBodyTextFrame, mainBodyTextFrame.getWidth() * TEXT_FRAME_PAD_WIDTH, mainBodyTextFrame.getHeight() * TEXT_FRAME_PAD_HEIGHT);
    }
@@ -52,7 +52,7 @@ public class PageContentExtractor {
                        stripper.getRulings()));
            }
        }
-    
+
        return textPositionSequencesPerPage;
    }

@@ -12,9 +12,9 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;

 import lombok.RequiredArgsConstructor;
@@ -25,10 +25,13 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RulingCleaningService {

-    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
+    private static final float THRESHOLD = 6;
+
+
+    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {

        if (!rulings.isEmpty()) {
-            snapPoints(rulings, minCharWidth, maxCharHeight);
+            snapPoints(rulings);
        }

        List<Ruling> vrs = new ArrayList<>();
@@ -53,14 +56,11 @@ public class RulingCleaningService {
        }
        List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);

-        return CleanRulings.builder()
-                .vertical(verticalRulingLines)
-                .horizontal(horizontalRulingLines)
-                .build();
+        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
    }


-    public void snapPoints(List<? extends Line2D.Float> rulings, float xThreshold, float yThreshold) {
+    public void snapPoints(List<? extends Line2D.Float> rulings) {

        // collect points and keep a Line -> p1,p2 map
        Map<Line2D.Float, Point2D[]> linesToPoints = new HashMap<>();
@@ -81,7 +81,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < xThreshold) {
+            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -108,7 +108,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < yThreshold) {
+            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -1,146 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BinaryOperator;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collector;
-import java.util.stream.Stream;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class SectionGridCreatorService {
-
-    public SectionGrid createSectionGrid(Document document) {
-
-        Map<Integer, List<SectionRectangle>> sectionBBox = document.streamAllSubNodesOfType(NodeType.SECTION).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> paragraphBBox = document.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> headlineBBox = document.streamAllSubNodesOfType(NodeType.HEADLINE).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> tableBBox = document.streamAllSubNodesOfType(NodeType.TABLE).map(node -> (Table) node).collect(new TableGridCollector());
-        var sectionGrid = new SectionGrid();
-
-        sectionGrid.setRectanglesPerPage(mergeMapsByConcatenatingLists(//
-                mergeMapsByConcatenatingLists(paragraphBBox, headlineBBox), //
-                mergeMapsByConcatenatingLists(sectionBBox, tableBBox)));
-
-        return sectionGrid;
-    }
-
-
-    private static abstract class GridCollector<T> implements Collector<T, Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> {
-
-        @Override
-        public Supplier<Map<Integer, List<SectionRectangle>>> supplier() {
-
-            return HashMap::new;
-        }
-
-
-        @Override
-        public Function<Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> finisher() {
-
-            return Function.identity();
-        }
-
-
-        @Override
-        public BinaryOperator<Map<Integer, List<SectionRectangle>>> combiner() {
-
-            return SectionGridCreatorService::mergeMapsByConcatenatingLists;
-        }
-
-
-        @Override
-        public Set<Characteristics> characteristics() {
-
-            return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
-        }
-
-    }
-
-    private static class TableGridCollector extends GridCollector<Table> {
-
-        @Override
-        public BiConsumer<Map<Integer, List<SectionRectangle>>, Table> accumulator() {
-
-            return (map, table) -> table.getPages()
-                    .forEach(page -> map.merge(page.getNumber(), List.of(toSectionRectangle(table, page, table.getPages().size())), SectionGridCreatorService::concatLists));
-        }
-
-
-        private static SectionRectangle toSectionRectangle(Table table, Page page, int numberOfParts) {
-
-            Rectangle2D rect = table.getBBox().get(page);
-            List<CellRectangle> tableCellRectangles = table.streamTableCells()
-                    .map(TableCell::getBBox)
-                    .map(map -> map.get(page))
-                    .filter(Objects::nonNull)
-                    .map(rectangle2D -> new CellRectangle(new Point((float) rectangle2D.getX(), (float) rectangle2D.getY()),
-                            (float) rectangle2D.getWidth(),
-                            (float) rectangle2D.getHeight()))
-                    .toList();
-            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()),
-                    (float) rect.getWidth(),
-                    (float) rect.getHeight(),
-                    1,
-                    numberOfParts,
-                    tableCellRectangles);
-        }
-
-    }
-
-    private static class SectionGridCollector extends GridCollector<Map<Page, Rectangle2D>> {
-
-        @Override
-        public BiConsumer<Map<Integer, List<SectionRectangle>>, Map<Page, Rectangle2D>> accumulator() {
-
-            return (mapToKeep, mapToMerge) -> mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page.getNumber(),
-                    List.of(toSectionRectangle(rectangle, mapToMerge.values().size())),
-                    SectionGridCreatorService::concatLists));
-
-        }
-
-
-        private static SectionRectangle toSectionRectangle(Rectangle2D rect, int numberOfParts) {
-
-            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()), (float) rect.getWidth(), (float) rect.getHeight(), 1, numberOfParts, null);
-        }
-
-    }
-
-
-    private static Map<Integer, List<SectionRectangle>> mergeMapsByConcatenatingLists(Map<Integer, List<SectionRectangle>> mapToKeep,
-                                                                                      Map<Integer, List<SectionRectangle>> mapToMerge) {
-
-        mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page, rectangle, SectionGridCreatorService::concatLists));
-        return mapToKeep;
-    }
-
-
-    private static List<SectionRectangle> concatLists(List<SectionRectangle> l1, List<SectionRectangle> l2) {
-
-        return Stream.concat(l1.stream(), l2.stream()).toList();
-    }
-
-}
@@ -5,9 +5,9 @@ import java.util.List;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;

@Service
 public class SimplifiedSectionTextService {
@@ -23,4 +23,5 @@ public class SimplifiedSectionTextService {

        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
    }
+
 }
@@ -12,7 +12,6 @@ import java.util.Set;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -66,6 +65,17 @@ public class TableExtractionService {
    };


+    public boolean contains(Cell cell, double x, double y, double w, double h) {
+
+        if (cell.isEmpty() || w <= 0 || h <= 0) {
+            return false;
+        }
+        double x0 = cell.getX();
+        double y0 = cell.getY();
+        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
+    }
+
+
    /**
     * Finds tables on a page and moves textblocks into cells of the found tables.
     * Note: This algorithm uses Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -79,16 +89,17 @@ public class TableExtractionService {
     * @param cleanRulings The lines used to build the table.
     * @param page         Page object that contains textblocks and statistics.
     */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page, LayoutParsingType layoutParsingType) {
+    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {

-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical(), layoutParsingType);
+        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());

        List<TextPageBlock> toBeRemoved = new ArrayList<>();

        for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
            TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
            for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
+                if (cell.hasMinimumSize() && contains(cell,
+                        textBlock.getPdfMinX(),
                        textBlock.getPdfMinY(),
                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -102,7 +113,7 @@ public class TableExtractionService {
        cells = new ArrayList<>(new HashSet<>(cells));
        DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);

-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).toList();
+        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);

        List<TablePageBlock> tables = new ArrayList<>();
        for (Rectangle area : spreadsheetAreas) {
@@ -135,16 +146,14 @@ public class TableExtractionService {
    }


-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, LayoutParsingType layoutParsingType) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

-        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
-            // TODO: breaks some tables, for example "1 Abamectin Prr.pdf" try to fix this upstream in RulingCleaningService
-            for (Ruling r : horizontalRulingLines) {
-                if (r.getX2() < r.getX1()) {
-                    double a = r.getX2();
-                    r.x2 = (float) r.getX1();
-                    r.x1 = (float) a;
-                }
+        // Fix for 211.pdf
+        for (Ruling r : horizontalRulingLines) {
+            if (r.getX2() < r.getX1()) {
+                double a = r.getX2();
+                r.x2 = (float) r.getX1();
+                r.x1 = (float) a;
            }
        }

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;

-
 // TODO: figure out, why this fails the build
 // import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;

@@ -33,9 +32,10 @@ public class TaasBlockificationService {
    private static final float INTERSECTS_Y_THRESHOLD = 4;// 2 * HEIGHT_PADDING // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
    private static final int X_GAP_SPLIT_CONSTANT = 50;
    public static final int X_ALIGNMENT_THRESHOLD = 1;
-    public static final int SMALL_Y_GAP_THRESHOLD = 5;
    public static final int NEGATIVE_X_GAP_THRESHOLD = -5;

+    private Pattern listIdentifier = Pattern.compile("^(?:(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)]))|\\uF0B7", Pattern.CASE_INSENSITIVE);
+

    /**
     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
@@ -80,16 +80,29 @@ public class TaasBlockificationService {
        List<TextPageBlock> currentTextBlocksToMerge = new LinkedList<>();
        textBlocksToMerge.add(currentTextBlocksToMerge);
        TextPageBlock previousTextBlock = null;
+        Float lastLineGap = null;
        for (TextPageBlock currentTextBlock : classificationTextBlocks) {
            if (previousTextBlock == null) {
                currentTextBlocksToMerge.add(currentTextBlock);
                previousTextBlock = currentTextBlock;
                continue;
            }
+
+            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
+            boolean isListIdentifier = listIdentifierPattern.find();
+
+            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+
+            boolean sameFont = previousTextBlock.getMostPopularWordFont()
+                    .equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
+//            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+
            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
-            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < SMALL_Y_GAP_THRESHOLD;
-            if (alignsXRight && smallYGap) {
+            boolean alignsXLeft = Math.abs(currentTextBlock.getPdfMinX() - previousTextBlock.getPdfMinX()) < X_ALIGNMENT_THRESHOLD;
+//            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < yGap;
+            if (yGap && sameFont && !isListIdentifier) {
                currentTextBlocksToMerge.add(currentTextBlock);
+
            } else {
                currentTextBlocksToMerge = new LinkedList<>();
                currentTextBlocksToMerge.add(currentTextBlock);
@@ -111,8 +124,9 @@ public class TaasBlockificationService {
            }
            alreadyMerged.add(textPageBlock);
            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
-                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
-                    .toList());
+                    textPageBlocks.stream()
+                            .filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2))
+                            .peek(alreadyMerged::add)).toList());
        }
        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
    }
@@ -155,8 +169,7 @@ public class TaasBlockificationService {
        while (itty.hasNext()) {
            TextPageBlock block = (TextPageBlock) itty.next();

-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(
-                    block.getMaxY(),
+            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
                previous.add(block);
@@ -170,8 +183,8 @@ public class TaasBlockificationService {


    private List<TextPageBlock> constructFineGranularTextPageBlocks(List<TextPositionSequence> textPositions,
-                                                                                        List<Ruling> horizontalRulingLines,
-                                                                                        List<Ruling> verticalRulingLines) {
+                                                                    List<Ruling> horizontalRulingLines,
+                                                                    List<Ruling> verticalRulingLines) {

        int indexOnPage = 0;
        List<TextPositionSequence> wordClusterToCombine = new ArrayList<>();
@@ -180,13 +193,12 @@ public class TaasBlockificationService {
        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
        TextPositionSequence prev = null;
        // TODO: make static final constant
-        var listIdentitifier = Pattern.compile("\\b(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)])", Pattern.CASE_INSENSITIVE);

        boolean wasSplitted = false;
        Float splitX1 = null;
        for (TextPositionSequence word : textPositions) {

-            Matcher listIdentifierPattern = listIdentitifier.matcher(word.toString());
+            Matcher listIdentifierPattern = listIdentifier.matcher(word.toString());

            boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
            boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());
@@ -5,7 +5,6 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -13,6 +12,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -63,16 +63,16 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
-        ) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
-        ) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -11,6 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RedactManagerClassificationService {

-
    public void classifyDocument(ClassificationDocument document) {

        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
@@ -52,14 +51,16 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -12,6 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -27,7 +27,6 @@ public class TaasClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
@@ -57,11 +56,13 @@ public class TaasClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation())) {
            textBlock.setClassification(PageBlockType.HEADER);
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -18,8 +18,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
@@ -31,6 +29,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

@@ -8,10 +8,10 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Objects;

+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;

 import lombok.experimental.UtilityClass;

@@ -110,6 +110,7 @@ public class SearchTextWithTextPositionFactory {
        return context.stringIdx - context.lastHyphenIdx < MAX_HYPHEN_LINEBREAK_DISTANCE;
    }

+
    private static List<Boundary> mergeToBoundaries(List<Integer> integers) {

        if (integers.isEmpty()) {
@@ -125,19 +126,21 @@ public class SearchTextWithTextPositionFactory {
            }
            end = current + 1;
        }
-        if (boundaries.isEmpty())
+        if (boundaries.isEmpty()) {
            boundaries.add(new Boundary(start, end));
+        }
        return boundaries;
    }


    private static void addTextPositionWithFontType(RedTextPosition currentTextPosition, String fontType, List<Integer> fontTypePositions, int stringIdx) {

-        if (currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
+        if (currentTextPosition.getFontName() != null && currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
            fontTypePositions.add(stringIdx);
        }
    }

+
    private boolean isLineBreak(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition) {

        return Objects.equals(currentTextPosition.getUnicode(), "\n") || isDeltaYLargerThanTextHeight(currentTextPosition, previousTextPosition);
@@ -163,17 +166,7 @@ public class SearchTextWithTextPositionFactory {

    private boolean isHyphen(String unicodeCharacter) {

-        return Objects.equals(unicodeCharacter, "-") || //
-                Objects.equals(unicodeCharacter, "~") || //
-                Objects.equals(unicodeCharacter, "‐") || //
-                Objects.equals(unicodeCharacter, "‒") || //
-                Objects.equals(unicodeCharacter, "⁻") || //
-                Objects.equals(unicodeCharacter, "−") || //
-                Objects.equals(unicodeCharacter, "﹣") || //
-                Objects.equals(unicodeCharacter, "゠") || //
-                Objects.equals(unicodeCharacter, "⁓") || //
-                Objects.equals(unicodeCharacter, "‑") || //
-                Objects.equals(unicodeCharacter, "\u00AD");
+        return false;
    }


@@ -11,12 +11,12 @@ import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TableMergingUtility;

 import lombok.experimental.UtilityClass;
@@ -8,15 +8,15 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import lombok.experimental.UtilityClass;
@@ -2,10 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;

 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -7,11 +7,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
@@ -1,18 +1,34 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.parsing;

-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.color.CMMException;
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.*;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
 import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
 import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
-import org.apache.pdfbox.contentstream.operator.state.*;
+import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
+import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
+import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
+import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
+import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
+import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
+import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSNumber;
@@ -21,11 +37,14 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.text.TextPosition;

-import java.awt.color.CMMException;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;

@Getter
@Slf4j
@@ -36,11 +55,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private final List<Ruling> graphicsPath = new ArrayList<>();
    @Setter
    protected PDPage pdpage;
-    private int minCharWidth;
-    private int maxCharWidth;
-    private int minCharHeight;
-    private int maxCharHeight;
-

    private float path_x;
    private float path_y;
@@ -73,7 +87,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        this.addOperator(new SetFontAndSize(this));
        this.addOperator(new SetLineWidth(this));

-
        addOperator(new BeginMarkedContentSequenceWithProperties(this));
 //        addOperator(new BeginMarkedContentSequence(this));
        addOperator(new EndMarkedContentSequence(this));
@@ -232,33 +245,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
            }

-            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
-            if (charWidth < minCharWidth) {
-                minCharWidth = charWidth;
-            }
-            if (charWidth > maxCharWidth) {
-                maxCharWidth = charWidth;
-            }
-
-            int charHeight = (int) textPositions.get(i).getHeightDir();
-            if (charHeight < minCharHeight) {
-                minCharHeight = charHeight;
-            }
-            if (charWidth > maxCharHeight) {
-                maxCharHeight = charHeight;
-            }
-
            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
                startIndex++;
                continue;
            }

            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj()) {
+            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -266,9 +261,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -278,13 +271,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                    .getUnicode()
                    .equals("\t")) && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {

                    // Remove false sequence ends (whitespaces)
-                    if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                            .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
+                    if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
                        for (TextPosition t : sublist) {
                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                        }
@@ -319,13 +309,30 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    }


+    public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {
+
+        return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
+    }
+
+
+    public boolean checkIfSequenceContainsOnlyWhitespaces(List<TextPosition> sublist) {
+
+        return !(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                .getUnicode()
+                .equals("\t")));
+    }
+
+
+    public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {
+
+        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
+                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
+    }
+
+
    @Override
    public String getText(PDDocument doc) throws IOException {

-        minCharWidth = Integer.MAX_VALUE;
-        maxCharWidth = 0;
-        minCharHeight = Integer.MAX_VALUE;
-        maxCharHeight = 0;
        textPositionSequences.clear();
        rulings.clear();
        graphicsPath.clear();
@@ -25,10 +25,23 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.regex.Pattern;

-import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.pdfbox.cos.COSDictionary;
@@ -46,6 +59,8 @@ import org.apache.pdfbox.text.TextPositionComparator;

 import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;

+import lombok.Getter;
+
 /**
 * This is just a copy except i only adjusted lines 594-607 cause this is a bug in Pdfbox.
 * see S416.pdf
@@ -194,40 +209,33 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    }


+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {

-    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
-    {
        PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
-        if (this.currentMarkedContents.isEmpty())
-        {
+        if (this.currentMarkedContents.isEmpty()) {
            this.markedContents.add(markedContent);
-        }
-        else
-        {
-            PDMarkedContent currentMarkedContent =
-                    this.currentMarkedContents.peek();
-            if (currentMarkedContent != null)
-            {
+        } else {
+            PDMarkedContent currentMarkedContent = this.currentMarkedContents.peek();
+            if (currentMarkedContent != null) {
                currentMarkedContent.addMarkedContent(markedContent);
            }
        }
        this.currentMarkedContents.push(markedContent);
    }

+
    @Override
-    public void endMarkedContentSequence()
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void endMarkedContentSequence() {
+
+        if (!this.currentMarkedContents.isEmpty()) {
            this.currentMarkedContents.pop();
        }
    }


-    public void xobject(PDXObject xobject)
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void xobject(PDXObject xobject) {
+
+        if (!this.currentMarkedContents.isEmpty()) {
            this.currentMarkedContents.peek().addXObject(xobject);
        }
    }
@@ -635,7 +643,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                        var normalized = normalize(line);
 //                        normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent()

-
                        lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine);
                        writeLine(normalized, current.isParagraphStart);
                        line.clear();
@@ -914,8 +921,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    textList.add(text);
                }
            }
-            if (!this.currentMarkedContents.isEmpty())
-            {
+            if (!this.currentMarkedContents.isEmpty()) {
                this.currentMarkedContents.peek().addText(text);
            }
        }
@@ -1711,7 +1717,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
        int numberOfStrings = line.size();
        for (int i = 0; i < numberOfStrings; i++) {
            WordWithTextPositions word = line.get(i);
-            word.getTextPositions().sort(Comparator.comparing(TextPosition::getX));
+            word.getTextPositions().sort(Comparator.comparing(TextPosition::getXDirAdj));
            writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
            if (i < numberOfStrings - 1) {
                writeWordSeparator();
@@ -2102,7 +2108,9 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            return endParagraphWritten;
        }

-        public void setEndParagraphWritten(){
+
+        public void setEndParagraphWritten() {
+
            endParagraphWritten = true;
        }

@@ -2145,7 +2153,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            this.isHangingIndent = true;
        }

-
    }

 }
@@ -1,10 +1,13 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.visualization;

+import java.awt.Color;
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import org.apache.pdfbox.cos.COSDictionary;
@@ -30,6 +33,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.visualization.
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.LayoutGrid;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;

 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -40,7 +44,6 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class ViewerDocumentService {

-
    private static final String LAYER_NAME = "Layout grid";
    private static final int FONT_SIZE = 10;
    public static final float LINE_WIDTH = 1f;
@@ -49,13 +52,18 @@ public class ViewerDocumentService {


    @SneakyThrows
-    public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream, boolean layerVisibilityDefaultValue) {
+    public void createViewerDocument(PDDocument pdDocument,
+                                     Document document,
+                                     OutputStream outputStream,
+                                     Map<Integer,List<TableExtractorCells>> extractedTableCells,
+                                     boolean layerVisibilityDefaultValue) {

        LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
        // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
        // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
+        PDOptionalContentGroup visualLayoutParsingLayer = addLayerToDocument(pdDocument, dictionariesToUpdate, true);
        PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);

        for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
@@ -72,44 +80,25 @@ public class ViewerDocumentService {
            assert pageNumber == visualizationsOnPage.getPageNumber();
            // We need to append to the content stream, otherwise the content could be overlapped by following content.
            try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true)) {
-
-                contentStream.beginMarkedContent(COSName.OC, layer);
+                contentStream.beginMarkedContent(COSName.OC, visualLayoutParsingLayer);
                contentStream.saveGraphicsState();

                contentStream.setLineWidth(LINE_WIDTH);
-                for (ColoredLine coloredLine : visualizationsOnPage.getColoredLines()) {
-                    contentStream.setStrokingColor(coloredLine.color());
-                    contentStream.moveTo((float) coloredLine.line().getX1(), (float) coloredLine.line().getY1());
-                    contentStream.lineTo((float) coloredLine.line().getX2(), (float) coloredLine.line().getY2());
+                for (TableExtractorCells tableCells : extractedTableCells.get(pageNumber)) {
+                    contentStream.setStrokingColor(new Color(0xB700FF));
+                    contentStream.addRect((float) tableCells.getX0(), (float) tableCells.getY0(), (float) tableCells.getWidth(), (float) tableCells.getHeight());
                    contentStream.stroke();
-                }
-                for (ColoredRectangle coloredRectangle : visualizationsOnPage.getColoredRectangles()) {
-                    contentStream.setStrokingColor(coloredRectangle.color());
-                    Rectangle2D r = coloredRectangle.rectangle2D();
-                    contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
-                    contentStream.stroke();
-                }
-                for (FilledRectangle filledRectangle : visualizationsOnPage.getFilledRectangles()) {
-                    contentStream.setNonStrokingColor(filledRectangle.color());
-                    PDExtendedGraphicsState graphicsState = new PDExtendedGraphicsState();
-                    graphicsState.setNonStrokingAlphaConstant(filledRectangle.alpha());
-                    contentStream.setGraphicsStateParameters(graphicsState);
-                    Rectangle2D r = filledRectangle.rectangle2D();
-                    contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
-                    contentStream.fill();
-                }
-                for (PlacedText placedText : visualizationsOnPage.getPlacedTexts()) {
                    contentStream.setFont(font, FONT_SIZE);
                    contentStream.beginText();
                    Matrix textMatrix = new Matrix((float) textDeRotationMatrix.getScaleX(),
                            (float) textDeRotationMatrix.getShearX(),
                            (float) textDeRotationMatrix.getShearY(),
                            (float) textDeRotationMatrix.getScaleY(),
-                            (float) placedText.lineStart().getX(),
-                            (float) placedText.lineStart().getY());
-                    textMatrix.translate(-((font.getStringWidth(placedText.text()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
+                            tableCells.getX0() ,
+                            tableCells.getY0());
+                    textMatrix.translate(-((font.getStringWidth(tableCells.getLabel()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
                    contentStream.setTextMatrix(textMatrix);
-                    contentStream.showText(placedText.text());
+                    contentStream.showText(tableCells.getLabel());
                    contentStream.endText();
                }
                contentStream.restoreGraphicsState();
@@ -1,17 +1,20 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import lombok.experimental.UtilityClass;
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.apache.pdfbox.text.TextPosition;

-import java.awt.geom.Rectangle2D;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.experimental.UtilityClass;

@UtilityClass
 public class MarkedContentUtils {
@@ -19,10 +22,11 @@ public class MarkedContentUtils {
    public static final String HEADER = "Header";
    public static final String FOOTER = "Footer";

+
    public List<Rectangle2D> getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {

        if (markedContents == null) {
-            return null;
+            return Collections.emptyList();
        }

        var markedContentByYPosition = markedContents.stream()
@@ -30,26 +34,30 @@ public class MarkedContentUtils {
                .filter(m -> m.getProperties() != null)
                .filter(m -> m.getProperties().getItem("Subtype") != null)
                .filter(m -> ((COSName) m.getProperties().getItem("Subtype")).getName().equals(subtype))
-                .map(PDMarkedContent::getContents).flatMap(Collection::stream)
+                .map(PDMarkedContent::getContents)
+                .flatMap(Collection::stream)
                .filter(t -> t instanceof TextPosition)
                .map(t -> (TextPosition) t)
                .filter(t -> !t.getUnicode().equals(" "))
                .collect(Collectors.groupingBy(TextPosition::getY));

        if (markedContentByYPosition.isEmpty()) {
-            return null;
+            return Collections.emptyList();
        }

-        return markedContentByYPosition.values().stream()
-                .map(textPositions -> new TextPositionSequence(textPositions.stream()
-                        .toList(), 0, true)
-                        .getRectangle())
-                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList());
+        return markedContentByYPosition.values()
+                .stream()
+                .map(textPositions -> new TextPositionSequence(textPositions.stream().toList(), 0, true).getRectangle())
+                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight())))
+                .collect(Collectors.toList());
    }


    public boolean intersects(TextPageBlock textBlock, Map<String, List<Rectangle2D>> markedContentBboxPerType, String type) {
-        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).stream().anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
+
+        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type)
+                .stream()
+                .anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
    }

 }
@@ -41,11 +41,14 @@ public class RectangleTransformations {

        return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DBBoxCollector());
    }
+
+
    public static Collector<Rectangle2D, Rectangle2DBBoxCollector.BBox, Rectangle2D> collectBBox() {

        return new Rectangle2DBBoxCollector();
    }

+
    public static PDRectangle toPDRectangleBBox(List<Rectangle> rectangles) {

        Rectangle2D rectangle2D = RectangleTransformations.rectangleBBox(rectangles);
@@ -70,6 +73,7 @@ public class RectangleTransformations {
        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
    }

+
    public static Rectangle2D rectangleBBox(List<Rectangle> rectangles) {

        return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DBBoxCollector());
@@ -84,6 +88,7 @@ public class RectangleTransformations {
                -redactionLogRectangle.getHeight());
    }

+
    public static Rectangle2D toRectangle2D(PDRectangle rectangle) {

        return new Rectangle2D.Double(rectangle.getLowerLeftX(), rectangle.getLowerLeftY(), rectangle.getWidth(), rectangle.getHeight());
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 import java.util.List;
 import java.util.stream.Collectors;

-
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

@@ -28,15 +28,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPosit
 *
 * @author Ben Litchfield
 */
-public class TextPositionSequenceComparator implements Comparator<TextPositionSequence>
-{
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
+
    @Override
-    public int compare(TextPositionSequence pos1, TextPositionSequence pos2)
-    {
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
        // only compare text that is in the same direction
        int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
-        if (cmp1 != 0)
-        {
+        if (cmp1 != 0) {
            return cmp1;
        }

@@ -54,19 +52,13 @@ public class TextPositionSequenceComparator implements Comparator<TextPositionSe
        float yDifference = Math.abs(pos1YBottom - pos2YBottom);

        // we will do a simple tolerance comparison
-        if (yDifference < .1 ||
-                pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
-                pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
-        {
+        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
            return Float.compare(x1, x2);
-        }
-        else if (pos1YBottom < pos2YBottom)
-        {
+        } else if (pos1YBottom < pos2YBottom) {
            return -1;
-        }
-        else
-        {
+        } else {
            return 1;
        }
    }
+
 }
@@ -19,8 +19,8 @@ dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.commons:storage-commons:2.40.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.14.0")
    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
    implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
@@ -14,7 +14,7 @@ import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfigur
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
-@Import({MetricsConfiguration.class, StorageAutoConfiguration.class,  LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
+@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

@@ -26,6 +26,7 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
@@ -50,7 +51,7 @@ public class BdrJsonBuildTest extends AbstractTest {
            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
                    pdDocument,
                    new ImageServiceResponse(),
-                    new TableServiceResponse()));
+                    new TableServiceResponse(), new TableExtractorResponse()));
        }
    }

@@ -33,6 +33,7 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
@@ -98,7 +99,7 @@ public class HeadlinesGoldStandardIntegrationTest {
        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                Loader.loadPDF(pdfFileResource.getFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse()));
+                new TableServiceResponse(), new TableExtractorResponse()));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -16,6 +16,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
@@ -58,7 +59,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                Loader.loadPDF(filename.toFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse()));
+                new TableServiceResponse(), new TableExtractorResponse()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -47,7 +47,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentTest {
    @Disabled
    public void visualizeCraftedDocument() {

-        String filename = "files/crafted document.pdf";
+        String filename = "files/1 Abamectin_prr.pdf";
        visualizePdf(filename);
    }

@@ -53,7 +53,7 @@ public class ExtractMarkedContentTest {
    @SneakyThrows
    public void testExtractTestWPhromma() throws IOException {
        System.out.printf("\n\n===\n%s\n===\n", "testWPhromma.pdf");
-            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
+        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {

            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();

@@ -70,6 +70,8 @@ public class ExtractMarkedContentTest {

            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
            showStructure(root, markedContents);
+            document.close();
+        }
    }

    /**
@@ -87,7 +89,8 @@ public class ExtractMarkedContentTest {
    @Test
    public void testExtractResMultipage() throws IOException {
        System.out.printf("\n\n===\n%s\n===\n", "res_multipage.pdf");
-            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
+
+        try(PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {

            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();

@@ -104,6 +107,8 @@ public class ExtractMarkedContentTest {

            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
            showStructure(root, markedContents);
+            document.close();
+        }
    }

    /**
@@ -121,7 +126,7 @@ public class ExtractMarkedContentTest {
    @Test
    public void testExtractDailyReport() throws IOException {
        System.out.printf("\n\n===\n%s\n===\n", "Daily Report.pdf");
-            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
+        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {

            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();

@@ -138,7 +143,9 @@ public class ExtractMarkedContentTest {

            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
            showStructure(root, markedContents);
+            document.close();
        }
+    }

    /**
     * @see #testExtractTestWPhromma()
@@ -1,36 +1,156 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

+import static com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw.drawRectangle2DList;
+
+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
+import java.io.File;
 import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;

 import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.json.JSONArray;
+import org.json.JSONObject;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
+import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Rectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTable;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.GapDetectionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.GapsAcrossLinesService;
+import com.knecon.fforesight.service.layoutparser.processor.services.LineDetectionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;

 public class ViewerDocumentTest extends BuildDocumentTest {

+    @Autowired
+    private SectionsBuilderService sectionsBuilderService;
+
+    @Autowired
+    private RedactManagerClassificationService redactManagerClassificationService;
+
+    @Autowired
+    private ObjectMapper objectMapper;
+
    @Test
-    @Disabled
    @SneakyThrows
    public void testViewerDocument() {

+        String fileName = "files/bdr/notMergedParagraphs.pdf";
+        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
        LayoutGridService layoutGridService = new LayoutGridService();
        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
-        String fileName = "files/bdr/notMergedParagraphs.pdf";
        Document document = buildGraph(fileName, LayoutParsingType.TAAS);
-        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
        try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getFile()); var out = new FileOutputStream(tmpFileName)) {
-            viewerDocumentService.createViewerDocument(pdDocument, document, out, true);
+            viewerDocumentService.createViewerDocument(pdDocument, document, out, null,true);
        }
    }

+    @Test
+    @SneakyThrows
+    public void testLayoutParsingServiceResults() {
+        String tableSourceFileName ="C:\\Users\\YannikHampe\\Downloads\\b28d9a22b674906813f12b86dda33202.EXTRACTED_TABLES.json\\b28d9a22b674906813f12b86dda33202.EXTRACTED_TABLES.json";
+        Path pdfFileResource = Path.of("C:\\Users\\YannikHampe\\Downloads\\2009-1048395_50pages_tables.pdf");
+        String tmpFileName = "C:/Users/YANNIK~1/AppData/Local/Temp/rectangles."+pdfFileResource.getFileName();
+        PDDocument pdDocument = Loader.loadPDF(pdfFileResource.toFile());
+
+        try (InputStream inputStream = Files.newInputStream(Path.of(tableSourceFileName))) {
+            TableExtractorResponse tableExtractorResponse = objectMapper.readValue(inputStream, TableExtractorResponse.class);
+            tableExtractorResponse.getData().forEach(data -> {
+                List<TableCells> tableCells = convertTableCells(data.getTables());
+            });
+            inputStream.close();
+        }
+        try (var out = new FileOutputStream(tmpFileName)) {
+            pdDocument.save(out);
+            pdDocument.close();
+        }
+
+    }
+
+    public List<TableCells> convertTableCells(List<ExtractedTable> tableObjects) {
+
+        List<TableCells> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            System.out.println(t.getTable().getLabel());
+            TableCells tableCells = new TableCells();
+            tableCells.setX0(t.getTable().getBbox().get(0));
+            tableCells.setX1(t.getTable().getBbox().get(2));
+            tableCells.setY0(t.getTable().getBbox().get(1));
+            tableCells.setY1(t.getTable().getBbox().get(3));
+            tableCells.setWidth(tableCells.getX1()- tableCells.getX0());
+            tableCells.setHeight(tableCells.getY1()- tableCells.getY0());
+            parsedTableCells.add(tableCells);
+            t.getObjects().forEach(o -> {
+                System.out.println(o.getLabel());
+                TableCells objectCell = new TableCells();
+                objectCell.setX0(t.getTable().getBbox().get(0));
+                objectCell.setX1(t.getTable().getBbox().get(2));
+                objectCell.setY0(t.getTable().getBbox().get(1));
+                objectCell.setY1(t.getTable().getBbox().get(3));
+                objectCell.setWidth(objectCell.getX1()- objectCell.getX0());
+                objectCell.setHeight(objectCell.getY1()- objectCell.getY0());
+                parsedTableCells.add(objectCell);
+            });
+        });
+
+        return parsedTableCells;
+
+    }
+
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                originDocument,
+                new ImageServiceResponse(),
+                new TableServiceResponse(), new TableExtractorResponse());
+
+        redactManagerClassificationService.classifyDocument(classificationDocument);
+
+        sectionsBuilderService.buildSections(classificationDocument);
+
+        return classificationDocument;
+    }
+
 }
@@ -27,14 +27,14 @@ public class TextPositionSequenceTest {
            + "  \"pageWidth\": 600\n"  //
            + "}";

-    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper(), new StorageProperties());
+    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper());


    @Test
    @SneakyThrows
    public void testDeserializationWithJackson() {

-        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithJackson(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+        TextPositionSequence textPositionSequence = objectSerializer.deserialize(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
                TextPositionSequence.class);

        assertPropertiesAfterJsonDeserialization(textPositionSequence);
@@ -1,5 +1,27 @@
 package com.knecon.fforesight.service.layoutparser.server.segmentation;

+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.awt.geom.Rectangle2D;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -11,23 +33,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePag
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+
 import lombok.SneakyThrows;
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
-import java.awt.geom.Rectangle2D;
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.assertj.core.api.Assertions.assertThat;

 public class PdfSegmentationServiceTest extends AbstractTest {

@@ -50,12 +62,13 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    private SectionsBuilderService sectionsBuilderService;


-    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+    @SneakyThrows
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument, TableServiceResponse tableServiceResponse) {

-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                originDocument,
                new ImageServiceResponse(),
-                new TableServiceResponse());
+                tableServiceResponse, new TableExtractorResponse());

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -65,6 +78,58 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @SneakyThrows
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+
+        return buildClassificationDocument(originDocument, new TableServiceResponse());
+    }
+
+
+    @Test
+    public void tablesToHtmlDebugger() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+
+        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");
+
+    }
+
+
+    @Test
+    public void tablesToHtmlDebuggerWithCVResponse() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
+        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
+        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()), tableServiceResponse);
+
+        toHtml(document, "/tmp/ScanRotationBorder.html");
+    }
+
+
+    @Test
+    public void testScanRotationBorderIsIgnored() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
+        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
+        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()), tableServiceResponse);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+
+        // Quality of the table parsing is not good, because the file is rotated at scanning.
+        // We only asset that the table border is not the page border.
+        tables.forEach(table -> {
+            assertThat(table.getMinX()).isGreaterThan(75);
+            assertThat(table.getMaxX()).isLessThan(512);
+        });
+    }
+
+
    @Test
    @SneakyThrows
    public void testMapping() {
@@ -155,7 +220,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


-    @Test
+    @Test // Non-sense test
    public void testDoc56Page170() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
@@ -166,8 +231,25 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        validateTable(document, 0, 1, 1, 0, 0);
        validateTable(document, 1, 2, 2, 0, 0);
-        validateTable(document, 2, 7, 20, 0, 140);
-        validateTable(document, 3, 8, 31, 0, 170);
+        validateTable(document, 2, 6, 20, 0, 0);
+        validateTable(document, 3, 7, 31, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc211() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/211.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 5, 4, 0, 0);
+        validateTable(document, 1, 5, 15, 14, 0);
+        validateTable(document, 2, 5, 14, 11, 0);
+        validateTable(document, 3, 5, 3, 0, 0);

    }

@@ -181,7 +263,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        validateTableSize(document, 1);

-        validateTable(document, 0, 8, 8, 0, 2);
+        validateTable(document, 0, 8, 8, 0, 0);

        List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
                        "Author, date",
@@ -191,18 +273,18 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                        "Method meets analytical validation criteria",
                        "Remarks (in case validation criteria are not met)",
                        "Acceptability of the method"),
-                Arrays.asList("",
+                Arrays.asList("Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
                        "Evans P.G. 2001 TMJ4569B, VV-323245",
                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
-                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
                        "Y",
                        "N/A",
@@ -220,6 +302,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

+        toHtml(document, "/tmp/html.html");
+
        validateTableSize(document, 4);

        validateTable(document, 0, 3, 2, 0, 0);
@@ -231,17 +315,29 @@ public class PdfSegmentationServiceTest extends AbstractTest {


    @Test
+    @Disabled // FIXME Fake Redactions leads to more cells, no solution for this currently
    public void testDocA20622APartB9Page185() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");

        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

-        validateTableSize(document, 2);
+        validateTableSize(document, 1);

-        validateTable(document, 0, 5, 5, 0, 23);
-        validateTable(document, 1, 11, 9, 0, 36);
+        validateTable(document, 0, 7, 4, 0, 0);
+    }

+
+    @Test
+    public void testDocA20622APartB9Page185FixedDoc() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185_fixed.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 7, 4, 0, 0);
    }


@@ -328,7 +424,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
-        validateTable(document, 0, 10, 6, 0, 1);
+        validateTable(document, 0, 10, 6, 0, 0);

    }

@@ -450,8 +546,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        validateTableSize(document, 2);

-        validateTable(document, 0, 6, 8, 0, 2);
-        validateTable(document, 1, 6, 8, 0, 1);
+        validateTable(document, 0, 6, 8, 0, 0);
+        validateTable(document, 1, 6, 8, 0, 0);

    }

@@ -484,12 +580,37 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @SneakyThrows
+    private void toHtml(ClassificationDocument document, String filename) {
+
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+        StringBuilder sb = new StringBuilder();
+
+        int currentPage = 1;
+        for (var table : tables) {
+            if (currentPage != table.getPage()) {
+                currentPage = table.getPage();
+                sb.append("---------------------- Page ").append(currentPage).append("--------------\n");
+            }
+            sb.append("\n\n");
+            sb.append(table.getTextAsHtml());
+        }
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(Path.of(filename).toFile())) {
+            fileOutputStream.write(sb.toString().getBytes());
+        }
+    }
+
+
    private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {

        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
        List<List<Cell>> rows = table.getRows();
        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();

+        for (List<Cell> row : table.getRows()) {
+            row.forEach(r -> System.out.println(r.toString()));
+        }
        assertThat(emptyCellsFoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);

        assertThat(table.getColCount()).isEqualTo(colCount);
@@ -1,21 +1,40 @@
 package com.knecon.fforesight.service.layoutparser.server.services;

+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;

+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;

-public class RulingCleaningServiceTest {
+public class RulingCleaningServiceTest extends BuildDocumentTest {

    @Test
 //    @Disabled
@@ -25,13 +44,96 @@ public class RulingCleaningServiceTest {
        String fileName = "files/211.pdf";
        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
-        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
-
        RulingCleaningService rulingCleaningService = new RulingCleaningService();
+        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
        List<CleanRulings> cleanRulingsPerPage = new LinkedList<>();
        for (PageContents pageContent : pageContents) {
-            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings(), 8, 20));
+            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings()));
+        }
+
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testTableExtraction() {
+
+        LayoutGridService layoutGridService = new LayoutGridService();
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
+
+        ClassPathResource resource = new ClassPathResource("files");
+        List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
+                .filter(path -> path.getFileName().toString().endsWith(".pdf"))
+                .map(Path::toAbsolutePath)
+                .map(Path::toString)
+                .toList();
+
+        for (int i = 0; i < pdfFileNames.size(); i++) {
+            writeJsons(Path.of(pdfFileNames.get(i)));
        }
    }

+
+    @SneakyThrows
+    private void writeJsons(Path filename) {
+
+        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                Loader.loadPDF(filename.toFile()),
+                new ImageServiceResponse(),
+                new TableServiceResponse(), new TableExtractorResponse()));
+        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                Loader.loadPDF(filename.toFile()),
+                new ImageServiceResponse(),
+                new TableServiceResponse(), new TableExtractorResponse()));
+        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
+        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
+        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
+            String tmpFileNameBefore = "C:/Users/YANNIK~1/AppData/Local/Temp/before." + filename.getFileName().toString();
+            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
+                PdfDraw.drawDocumentGraph(pdDocument, documentGraphBefore);
+                pdDocument.save(tmpFileNameBefore);
+            }
+            String tmpFileNameAfter = "C:/Users/YANNIK~1/AppData/Local/Temp/after." + filename.getFileName().toString();
+            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
+                PdfDraw.drawDocumentGraph(pdDocument, documentGraphAfter);
+                pdDocument.save(tmpFileNameAfter);
+
+            }
+        }
+    }
+
+
+    @SneakyThrows
+    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2, String pdfName) {
+
+        List listStructure1 = structure1.streamAllEntries()
+                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
+                .map(DocumentStructure.EntryData::getProperties)
+                .map(properties -> {
+                    var builder = Table.builder();
+                    PropertiesMapper.parseTableProperties(properties, builder);
+                    return builder.build();
+                })
+                .toList();
+
+        List listStructure2 = structure2.streamAllEntries()
+                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
+                .map(DocumentStructure.EntryData::getProperties)
+                .map(properties -> {
+                    var builder = Table.builder();
+                    PropertiesMapper.parseTableProperties(properties, builder);
+                    return builder.build();
+                })
+                .toList();
+
+        for (int i = 0; i < listStructure1.size(); i++) {
+            Table tableNode1 = (Table) listStructure1.get(i);
+            Table tableNode2 = (Table) listStructure2.get(i);
+            if (tableNode1.getNumberOfRows() != tableNode2.getNumberOfRows() || tableNode1.getNumberOfCols() != tableNode2.getNumberOfCols()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
 }
@@ -1,8 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

-import java.io.InputStream;
-import java.util.Optional;
-
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
+import com.knecon.fforesight.service.layoutparser.server.Application;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+import com.knecon.fforesight.tenantcommons.TenantsClient;
+import lombok.SneakyThrows;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -12,21 +19,14 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.*;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;

-import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
-import com.knecon.fforesight.service.layoutparser.server.Application;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-import com.knecon.fforesight.tenantcommons.TenantsClient;
+import java.io.InputStream;
+import java.util.Optional;

-import lombok.SneakyThrows;
+import javax.swing.text.html.Option;

@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -48,12 +48,13 @@ public abstract class AbstractTest {
    protected final static String ORIGIN_FILE_ID = "origin";
    protected final static String TABLE_FILE_ID = "table";
    protected final static String IMAGE_FILE_ID = "image";
+
+    protected final static String TABLE_EXTRACTOR_FILE_ID = "extractedTable";
    protected final static String STRUCTURE_FILE_ID = "structure";
    protected final static String TEXT_FILE_ID = "texts";
    protected final static String POSITION_FILE_ID = "positions";
    protected final static String PAGES_FILE_ID = "pages";
    protected final static String TENANT_ID = "tenant";
-    protected final static String SECTION_GRID_ID = "section";
    protected final static String VIEWER_DOCUMENT_ID = "viewer";
    protected final static String SIMPLIFIED_ID = "simplified";

@@ -65,12 +66,12 @@ public abstract class AbstractTest {
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .tableExtractorFileId(Optional.of(TABLE_EXTRACTOR_FILE_ID))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
                .positionBlockFileStorageId(POSITION_FILE_ID)
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
-                .sectionGridStorageId(SECTION_GRID_ID)
                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }
@@ -93,7 +94,7 @@ public abstract class AbstractTest {
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(String file) {

-        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json");
+        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json","table_extractor_response/empty.json");
    }


@@ -111,32 +112,34 @@ public abstract class AbstractTest {
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .tableExtractorFileId(Optional.of(TABLE_EXTRACTOR_FILE_ID))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
                .positionBlockFileStorageId(POSITION_FILE_ID)
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
-                .sectionGridStorageId(SECTION_GRID_ID)
                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }

    @SneakyThrows
-    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile) {
+    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile, String tableExtractorResponseFile) {

        ClassPathResource pdfFileResource = new ClassPathResource(file);
        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
        ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
+        ClassPathResource tableExtractorResponseFileRessource = new ClassPathResource(tableExtractorResponseFile);

-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream());
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream(),tableExtractorResponseFileRessource.getInputStream());
    }


    @SneakyThrows
-    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream) {
+    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream, InputStream tableExtractorResponseFileStream) {

        storageService.storeObject(TenantContext.getTenantId(), IMAGE_FILE_ID, imageInfoStream);
        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(), TABLE_EXTRACTOR_FILE_ID, tableExtractorResponseFileStream);
        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);

        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
@@ -157,6 +160,14 @@ public abstract class AbstractTest {
    @ComponentScan("com.knecon.fforesight.service.layoutparser")
    public static class TestConfiguration {

+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
+        }
+
    }

 }
@@ -11,6 +11,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;

@@ -28,7 +29,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
        ClassPathResource fileResource = new ClassPathResource(filename);
        prepareStorage(filename);
        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream.readAllBytes())) {
-            return layoutParsingPipeline.parseLayout(layoutParsingType, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse());
+            return layoutParsingPipeline.parseLayout(layoutParsingType, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), new TableExtractorResponse());
        }
    }

@@ -44,7 +45,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
    protected Document buildGraph(String filename, LayoutParsingType layoutParsingType) {

        if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
-            prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json");
+            prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json","table_extractor_response/empty.json");
        } else {
            prepareStorage(filename);
        }
@@ -1,124 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server.utils;
-
-import static java.io.File.createTempFile;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.commons.io.IOUtils;
-import org.springframework.stereotype.Service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
-@Service
-public class FileSystemBackedStorageService implements StorageService {
-
-    private final Map<String, File> dataMap = new HashMap<>();
-
-
-    public FileSystemBackedStorageService() {
-
-    }
-
-
-    @SneakyThrows
-    @Override
-    public void downloadTo(String tenantId, String objectId, File destinationFile) {
-
-        var res = dataMap.get(objectId);
-        if (res == null) {
-            throw new StorageObjectDoesNotExist(new RuntimeException());
-        }
-
-        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
-    }
-
-
-    @Override
-    public void deleteObject(String tenantId, String objectId) {
-
-        dataMap.remove(objectId);
-    }
-
-
-    @Override
-    public boolean objectExists(String tenantId, String objectId) {
-
-        return dataMap.containsKey(objectId);
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
-
-        File tempFile = createTempFile("test", ".tmp");
-        getMapper().writeValue(new FileOutputStream(tempFile), any);
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    private ObjectMapper getMapper() {
-
-        return ObjectMapperFactory.create();
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
-
-        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
-            throw new StorageObjectDoesNotExist("Stored object not found");
-        }
-        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
-    }
-
-
-    public List<String> listPaths() {
-
-        return new ArrayList<>(dataMap.keySet());
-    }
-
-
-    public List<String> listFilePaths() {
-
-        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
-    }
-
-
-    @Override
-    @SneakyThrows
-    public void storeObject(String tenantId, String objectId, InputStream stream) {
-
-        File tempFile = createTempFile("test", ".tmp");
-
-        try (var fileOutputStream = new FileOutputStream(tempFile)) {
-            IOUtils.copy(stream, fileOutputStream);
-        }
-
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    public void clearStorage() {
-
-        this.dataMap.forEach((k, v) -> {
-            v.delete();
-        });
-        this.dataMap.clear();
-    }
-
-}
@@ -0,0 +1,8 @@
+{
+  "dossierId": "123",
+  "fileId": "123",
+  "operation": "table",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "TABLES.json.gz",
+  "extractedTableData": []
+}
@@ -4,5 +4,5 @@ gradle assemble

 buildNumber=${1:-1}

-gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$USER-$buildNumber
-echo "nexus.knecon.com:5001/red/${dir}-server-v1:$USER-$buildNumber"
+gradle bootBuildImage --cleanCache --publishImage -Pversion=layout-parser-yannik-$buildNumber --stacktrace
+echo "nexus.knecon.com:5001/red/${dir}-server-v1:table-extractor-yannik-$buildNumber"
Author	SHA1	Message	Date
yhampe	aa1e77dda3	RED-7375 table extractor prototype	2024-02-01 08:23:53 +01:00
yhampe	0c3b910088	RED-7375 table extractor prototype	2024-01-30 11:59:18 +01:00
yhampe	b4e5f2da2f	RED-7375 table extractor prototype added label changed colour of bounding boxes	2024-01-17 16:58:53 +01:00
yhampe	b47b187c8a	RED-7375: table extractor prototype added layer for table extractor results in ViewerDocumentService	2024-01-15 09:43:23 +01:00
yhampe	0d4800622d	RED-7375:Integrate Table Parsing Service * integrated table parsing service * for visualization simpy drawing the extracted table in ViewerDocumentService	2023-12-01 13:26:16 +01:00
yhampe	cc492bc50d	RED-7375:Integrate Table Parsing Service * fixed merging mistake	2023-11-29 10:06:05 +01:00
yhampe	1c3c5385e1	Merge remote-tracking branch 'origin/RED-7375' into RED-7375	2023-11-29 09:40:24 +01:00
yhampe	bce2558133	RED-7375:Integrate Table Parsing Service * fixed end2end test by introducing sample table extractor response	2023-11-29 09:39:58 +01:00
yhampe	2200574b6d	RED-7375:Integrate Table Parsing Service * disabled not needed test	2023-11-29 09:39:58 +01:00
yhampe	775c943f7e	RED-7375: * using ViewerDocumentService to draw TableExtractorResponse into documents	2023-11-29 09:39:53 +01:00
yhampe	b7fe6fd3c4	RED-7375:Integrate Table Parsing Service * fixed end2end test by introducing sample table extractor response	2023-11-29 09:37:47 +01:00
yhampe	3dd447ebef	RED-7375:Integrate Table Parsing Service * disabled not needed test	2023-11-29 08:41:31 +01:00
yhampe	f4e93ef03b	RED-7375: * using ViewerDocumentService to draw TableExtractorResponse into documents	2023-11-28 15:14:04 +01:00
Kilian Schüttler	30f060e36c	Merge branch 'enable-caching' into 'main' enable caching in build See merge request fforesight/layout-parser!84	2023-11-24 10:36:55 +01:00
Kilian Schuettler	53a5824e6c	enable caching in build	2023-11-24 10:24:50 +01:00
Dominique Eifländer	e2bcf971c9	Merge branch 'DM-589' into 'main' DM-589: Filter wrong detected cells that borders from rotation at scanning See merge request fforesight/layout-parser!83	2023-11-20 16:07:27 +01:00
Dominique Eifländer	dacc2f7f43	DM-589: Filter wrong detected cells that borders from rotation at scanning	2023-11-20 15:54:02 +01:00
Dominique Eifländer	144a9591a2	Merge branch 'TAAS-103-hotfix' into 'main' * added back in if statement See merge request fforesight/layout-parser!82	2023-11-16 12:48:48 +01:00
yhampe	207d9dec97	* added back in if statement * removed not needed commentar	2023-11-16 12:40:49 +01:00
Yannik Hampe	09ee90222e	Merge branch 'TAAS-103' into 'main' TAAS-103: Table Detection and rotated text See merge request fforesight/layout-parser!81	2023-11-16 09:13:41 +01:00
yhampe	1316a067fe	* removed double chechking for height of cell	2023-11-16 08:51:12 +01:00
yhampe	e203210ade	* removed not needed properties	2023-11-16 08:23:58 +01:00
yhampe	b25d46291a	* checkstyle	2023-11-16 08:12:47 +01:00
yhampe	84148d3b6e	* fixed tests	2023-11-16 07:51:08 +01:00
Dominique Eifländer	a6ba66b1aa	TAAS-103: Fixed values in wrong cells	2023-11-15 13:36:46 +01:00
yhampe	c3e69b2cdf	* fixed bug with incorrect empty cell count by adding threshhold to cell.contains	2023-11-15 10:44:47 +01:00
yhampe	f69331e7d8	*renamed page to firstPage in DocumentStructure and Table	2023-11-07 10:21:19 +01:00
yhampe	01493dc033	TAAS-103: Table Detection and rotated text * added page property to DocumentStructure to be able to get page of found tables * added a method to TableExtractionService to get the table area * added calculateMinCharWidthAndMaxCharHeightInsideTable to LayoutParsingPipeline to calculate the values based upon table area * refactored PDFLinesTextStripper for better readability *removed textMatrix from RedTextPosition as it is no longer needed	2023-11-07 08:47:28 +01:00
yhampe	459e0c8be7	TAAS-103:	2023-11-07 08:39:15 +01:00
Kilian Schüttler	1b1f777706	Merge branch 'RED-7806' into 'main' RED-7806 - Specific customer document cannot be processed See merge request fforesight/layout-parser!79	2023-10-25 10:50:34 +02:00
Corina Olariu	0e0a811f9d	RED-7806 - Specific customer document cannot be processed - add brackets	2023-10-25 11:36:54 +03:00
Corina Olariu	efa3d75479	RED-7806 - Specific customer document cannot be processed - check for font name null before using to avoid the NPE	2023-10-25 09:16:47 +03:00
Kilian Schüttler	9abdc6d44d	Merge branch 'RED-7434' into 'main' RED-7434 - Remove Section Grid entirely See merge request fforesight/layout-parser!78	2023-10-20 10:07:01 +02:00
Corina Olariu	3bab61c446	RED-7434 - Remove Section Grid entirely - remove sectionGrid relation (including SectionGridCreatorService) - update junit tests	2023-10-20 09:09:22 +03:00
Dominique Eifländer	d17517d3c3	Merge branch 'hotfix-bdr-doc' into 'main' hotfix: Fixed parsing for specific taas document See merge request fforesight/layout-parser!77	2023-10-18 16:12:15 +02:00
Dominique Eifländer	567cbc178b	hotfix: Fixed parsing for specific taas document	2023-10-17 15:52:19 +02:00
Kilian Schüttler	3c53772765	Merge branch 'RED-7759' into 'main' RED-7759: Upgraded storage-commons to newest windwos compatible version See merge request fforesight/layout-parser!76	2023-10-13 12:21:04 +02:00
Dominique Eifländer	8647cf5a18	RED-7759: Upgraded storage-commons to newest windwos compatible version	2023-10-13 12:15:22 +02:00
Kilian Schüttler	310c07b200	Merge branch 'RED-7607-WIP' into 'main' RED-7607 - Rotating pages leads to lost annotations (RM & DM) See merge request fforesight/layout-parser!75	2023-10-05 13:34:12 +02:00
Corina Olariu	daba0bf8a6	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - remove finally clause	2023-10-04 17:46:46 +03:00
Corina Olariu	3839de215c	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - rollback to getDir().getDegrees()	2023-10-04 15:27:13 +03:00
Corina Olariu	b4d68594f1	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - use rotation instead of getDir().getDegrees()	2023-10-04 14:22:15 +03:00
Corina Olariu	99ed331a1e	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - use getXDirAdj instead of getX - add fontSizeCounter for landscape pages also	2023-10-04 14:13:38 +03:00
Corina Olariu	f2c0991987	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - fix PMD findings	2023-10-04 14:09:46 +03:00