RED-7375 table extractor prototype

2024-02-01 08:23:53 +01:00 · 2024-01-30 11:59:18 +01:00 · 2024-01-17 16:58:53 +01:00 · 2024-01-15 09:43:23 +01:00 · 2023-12-01 13:26:16 +01:00 · 2023-11-29 10:06:05 +01:00
75 changed files with 795 additions and 707 deletions
@@ -24,8 +24,6 @@ tasks.named<Test>("test") {
    reports {
        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
    }
-    minHeapSize = "512m"
-    maxHeapSize = "2048m"
 }

 tasks.test {
@@ -20,6 +20,9 @@ public record LayoutParsingRequest(
        @NonNull String originFileStorageId,//
        @Schema(description = "Optional Path to the table extraction file.")//
        Optional<String> tablesFileStorageId,//
+
+        @Schema(description= "Optional Path to the the table parsing service file")
+        Optional<String> tableExtractorFileId,
        @Schema(description = "Optional Path to the image classification file.")//
        Optional<String> imagesFileStorageId,//

@@ -15,8 +15,8 @@ dependencies {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
-    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("com.iqser.red.commons:storage-commons:2.40.0")

    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -3,16 +3,13 @@ package com.knecon.fforesight.service.layoutparser.processor;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
-import java.io.File;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;

-import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
@@ -33,8 +30,11 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageB
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.TableExtractorResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
@@ -54,121 +54,102 @@ import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDF
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;

-import io.micrometer.observation.Observation;
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.annotation.Observed;
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

-@SuppressWarnings("PMD.CloseResource")
@Slf4j
@Service
@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class LayoutParsingPipeline {

-    ImageServiceResponseAdapter imageServiceResponseAdapter;
-    CvTableParsingAdapter cvTableParsingAdapter;
-    LayoutParsingStorageService layoutParsingStorageService;
-    SectionsBuilderService sectionsBuilderService;
-    TaasClassificationService taasClassificationService;
-    RedactManagerClassificationService redactManagerClassificationService;
-    DocuMineClassificationService docuMineClassificationService;
-    SimplifiedSectionTextService simplifiedSectionTextService;
-    BodyTextFrameService bodyTextFrameService;
-    RulingCleaningService rulingCleaningService;
-    TableExtractionService tableExtractionService;
-    TaasBlockificationService taasBlockificationService;
-    DocuMineBlockificationService docuMineBlockificationService;
-    RedactManagerBlockificationService redactManagerBlockificationService;
-    ViewerDocumentService viewerDocumentService;
-    ObservationRegistry observationRegistry;
+    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
+    private final CvTableParsingAdapter cvTableParsingAdapter;
+    private final LayoutParsingStorageService layoutParsingStorageService;
+    private final TableExtractorResponseAdapter tableExtractorResponseAdapter;
+    private final SectionsBuilderService sectionsBuilderService;
+    private final TaasClassificationService taasClassificationService;
+    private final RedactManagerClassificationService redactManagerClassificationService;
+    private final DocuMineClassificationService docuMineClassificationService;
+    private final SimplifiedSectionTextService simplifiedSectionTextService;
+    private final BodyTextFrameService bodyTextFrameService;
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;
+    private final ViewerDocumentService viewerDocumentService;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {

        long start = System.currentTimeMillis();
-        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

-        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-        File viewerDocumentFile = File.createTempFile("viewer_document", ".pdf");
+        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
+            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            }

-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            TableExtractorResponse tableExtractorResponse = new TableExtractorResponse();
+            if (layoutParsingRequest.tableExtractorFileId().isPresent()) {
+                tableExtractorResponse = layoutParsingStorageService.getExtractedTableFile(layoutParsingRequest.tableExtractorFileId().get());
+            }
+
+            TableServiceResponse tableServiceResponse = new TableServiceResponse();
+            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+            }
+
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+                    originDocument,
+                    imageServiceResponse,
+                    tableServiceResponse,
+                    tableExtractorResponse);
+            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+
+            int numberOfPages = originDocument.getNumberOfPages();
+
+            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+            Map<Integer, List<TableExtractorCells>> extractedTableCells = tableExtractorResponseAdapter.buildExtractedTablesPerPage(tableExtractorResponse);
+            try (var out = new ByteArrayOutputStream()) {
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out,extractedTableCells ,false);
+                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
+            }
+
+            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+            }
+
+            return LayoutParsingFinishedEvent.builder()
+                    .identifier(layoutParsingRequest.identifier())
+                    .numberOfPages(numberOfPages)
+                    .duration(System.currentTimeMillis() - start)
+                    .message(format("""
+                                    Layout parsing has finished in %.02f s.
+                                    identifiers: %s
+                                    %s
+                                    Files have been saved with Ids:
+                                    Structure: %s
+                                    Text: %s
+                                    Positions: %s
+                                    PageData: %s
+                                    Simplified Text: %s
+                                    Viewer Doc: %s""",
+                            ((float) (System.currentTimeMillis() - start)) / 1000,
+                            layoutParsingRequest.identifier(),
+                            buildSemanticNodeCountMessage(numberOfPages, documentGraph.buildSemanticNodeCounts()),
+                            layoutParsingRequest.structureFileStorageId(),
+                            layoutParsingRequest.textBlockFileStorageId(),
+                            layoutParsingRequest.positionBlockFileStorageId(),
+                            layoutParsingRequest.pageFileStorageId(),
+                            layoutParsingRequest.simplifiedTextStorageId(),
+                            layoutParsingRequest.viewerDocumentStorageId()))
+                    .build();
        }
-
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
-        }
-
-        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
-                originFile,
-                imageServiceResponse,
-                tableServiceResponse,
-                layoutParsingRequest.identifier().toString());
-        log.info("Building document graph for {}", layoutParsingRequest.identifier());
-
-        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
-
-        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
-
-        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
-        viewerDocumentService.createViewerDocument(originFile, documentGraph, viewerDocumentFile, false);
-        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
-            log.info("Building research document data for {}", layoutParsingRequest.identifier());
-            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
-            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
-        }
-
-        originFile.delete();
-        viewerDocumentFile.delete();
-
-        return LayoutParsingFinishedEvent.builder()
-                .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(documentGraph.getNumberOfPages())
-                .duration(System.currentTimeMillis() - start)
-                .message(format("""
-                                Layout parsing has finished in %.02f s.
-                                identifiers: %s
-                                %s
-                                Files have been saved with Ids:
-                                Structure: %s
-                                Text: %s
-                                Positions: %s
-                                PageData: %s
-                                Simplified Text: %s
-                                Viewer Doc: %s""",
-                        ((float) (System.currentTimeMillis() - start)) / 1000,
-                        layoutParsingRequest.identifier(),
-                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId(),
-                        layoutParsingRequest.simplifiedTextStorageId(),
-                        layoutParsingRequest.viewerDocumentStorageId()))
-                .build();
-
-    }
-
-
-    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
-
-        AtomicReference<Document> documentReference = new AtomicReference<>();
-
-        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
-            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
-        });
-
-        return documentReference.get();
    }


@@ -187,36 +168,22 @@ public class LayoutParsingPipeline {


    @SneakyThrows
-    @Observed(name = "LayoutParsingPipeline", contextualName = "parse-layout")
    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
-                                              File originFile,
+                                              PDDocument originDocument,
                                              ImageServiceResponse imageServiceResponse,
                                              TableServiceResponse tableServiceResponse,
-                                              String identifier) {
+                                              TableExtractorResponse tableExtractorResponse) {

-        PDDocument originDocument = openDocument(originFile);
-        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
-
+        Map<Integer, List<TableExtractorCells>> extractedTableCells = tableExtractorResponseAdapter.buildExtractedTablesPerPage(tableExtractorResponse);
        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

+        originDocument.setAllSecurityToBeRemoved(true);
        long pageCount = originDocument.getNumberOfPages();

        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-
-            if (pageNumber % 100 == 0) {
-                // re-open document every once in a while to save on RAM. This has no significant performance impact.
-                // This is due to PDFBox caching all images and some other stuff with Soft References. This dereferences them and forces the freeing of memory.
-                originDocument.close();
-                originDocument = openDocument(originFile);
-            }
-
-            if (pageNumber % 100 == 0 || pageNumber == pageCount || pageNumber == 1) {
-                log.info("Extracting text on Page {} for {}", pageNumber, identifier);
-            }
-
            classificationDocument.setPages(classificationPages);
            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
            PDPage pdPage = originDocument.getPage(pageNumber - 1);
@@ -266,42 +233,21 @@ public class LayoutParsingPipeline {
            classificationPages.add(classificationPage);
        }

-        originDocument.close();
-
-        log.info("Calculating BodyTextFrame for {}", identifier);
        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
-        log.info("Classify TextBlocks for {}", identifier);
+
        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }

-        log.info("Building Sections for {}", identifier);
        sectionsBuilderService.buildSections(classificationDocument);
        sectionsBuilderService.addImagesToSections(classificationDocument);
+
        return classificationDocument;
    }


-    private void addNumberOfPagesToTrace(int numberOfPages, long size) {
-
-        if (observationRegistry.getCurrentObservation() != null) {
-            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
-            observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileSize", String.valueOf(size));
-        }
-    }
-
-
-    @SneakyThrows
-    private PDDocument openDocument(File originFile) {
-
-        PDDocument document = Loader.loadPDF(originFile);
-        document.setAllSecurityToBeRemoved(true);
-        return document;
-    }
-
-
    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {

        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor;

+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -9,6 +11,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;

+import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;
@@ -20,10 +23,10 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Si
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

-import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -37,18 +40,16 @@ public class LayoutParsingStorageService {
    private final ObjectMapper objectMapper;


-    public PDDocument getOriginDocument(String storageId) throws IOException {
+    public PDDocument getOriginFile(String storageId) throws IOException {

-        return Loader.loadPDF(getOriginFile(storageId));
-    }
-
-
-    @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
-    public File getOriginFile(String storageId) throws IOException {
-
-        File tempFile = createTempFile("document", ".pdf");
-        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
-        return tempFile;
+        try (var originDocumentInputStream = getObject(storageId)) {
+            File tempFile = createTempFile("document", ".pdf");
+            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
+                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
+            }
+            return Loader.loadPDF(tempFile);
+        }
    }


@@ -63,6 +64,16 @@ public class LayoutParsingStorageService {
    }


+    public TableExtractorResponse getExtractedTableFile(String storageId) throws IOException {
+
+        try (InputStream inputStream = getObject(storageId)) {
+            TableExtractorResponse tableExtractorResponse = objectMapper.readValue(inputStream, TableExtractorResponse.class);
+            inputStream.close();
+            return tableExtractorResponse;
+        }
+    }
+
+
    public TableServiceResponse getTablesFile(String storageId) throws IOException {

        try (var tableClassificationStream = getObject(storageId)) {
@@ -74,7 +85,6 @@ public class LayoutParsingStorageService {
    }


-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
@@ -115,7 +125,6 @@ public class LayoutParsingStorageService {
    }


-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-simplified-text")
    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
@@ -133,10 +142,9 @@ public class LayoutParsingStorageService {


    @SneakyThrows
-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-viewer-document")
-    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, File out) {
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {

-        try (var in = new FileInputStream(out)) {
+        try (var in = new ByteArrayInputStream(out.toByteArray())) {

            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
        }
@@ -14,7 +14,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFre
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
-import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;

@Data
@RequiredArgsConstructor
@@ -19,4 +19,5 @@ public class PageContents {
    Rectangle2D cropBox;
    Rectangle2D mediaBox;
    List<Ruling> rulings;
+
 }
@@ -108,11 +108,13 @@ public class Boundary implements Comparable<Boundary> {
        return splitBoundaries;
    }

+
    public IntStream intStream() {

        return IntStream.range(start, end);
    }

+
    public static Boundary merge(Collection<Boundary> boundaries) {

        int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
@@ -10,6 +10,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.amazonaws.services.kms.model.NotFoundException;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -83,7 +84,7 @@ public class Document implements GenericSemanticNode {
    @Override
    public Headline getHeadline() {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build());
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
    }


@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Footer implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -67,14 +62,4 @@ public class Footer implements GenericSemanticNode {
        return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Header implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public boolean isLeaf() {
@@ -67,14 +62,4 @@ public class Header implements GenericSemanticNode {
        return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Headline implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -74,14 +69,4 @@ public class Headline implements GenericSemanticNode {
        return this;
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,9 +32,6 @@ public class Paragraph implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -65,14 +60,4 @@ public class Paragraph implements GenericSemanticNode {
        return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -37,9 +35,6 @@ public class Section implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -79,14 +74,4 @@ public class Section implements GenericSemanticNode {
                .orElseGet(() -> getParent().getHeadline());
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -398,11 +398,12 @@ public interface SemanticNode {
     */
    default Map<Page, Rectangle2D> getBBox() {

+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        if (isLeaf()) {
-            return getBBoxFromLeafTextBlock();
+            return getBBoxFromLeafTextBlock(bBoxPerPage);
        }

-        return getBBoxFromChildren();
+        return getBBoxFromChildren(bBoxPerPage);
    }


@@ -425,31 +426,25 @@ public interface SemanticNode {

    /**
     * TODO: this produces unwanted results for sections spanning multiple columns.
-     * Computes the Union of the bounding boxes of all children recursively.
+     *
+     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of the BoundingBoxes of all children
     */
-    private Map<Page, Rectangle2D> getBBoxFromChildren() {
+    private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
-        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
-        for (Page page : pages) {
-            Rectangle2D bBoxOnPage = childrenBBoxes.stream()
-                    .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
-                    .map(childBboxPerPage -> childBboxPerPage.get(page))
-                    .collect(RectangleTransformations.collectBBox());
-            bBoxPerPage.put(page, bBoxOnPage);
-        }
-        return bBoxPerPage;
+        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
+            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
+            return map2;
+        }).orElse(bBoxPerPage);
    }


    /**
+     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of all BoundingBoxes of the TextBlock of this node
     */
-    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
+    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
        return bBoxPerPage;
@@ -2,12 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

 import static java.lang.String.format;

-import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -42,8 +40,6 @@ public class Table implements SemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;

    /**
     * Streams all entities in this table, that appear in a row, which contains any of the provided strings.
@@ -211,6 +207,7 @@ public class Table implements SemanticNode {
        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
    }

+
    /**
     * Streams all TableCells row-wise and filters them with header == true.
     *
@@ -315,12 +312,5 @@ public class Table implements SemanticNode {

        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
    }
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {

-        if (bBoxCache == null) {
-            bBoxCache = SemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
 }
@@ -109,10 +109,7 @@ public class AtomicTextBlock implements TextBlock {
    }


-    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData,
-                                                          DocumentPositionData documentPositionData,
-                                                          SemanticNode parent,
-                                                          Page page) {
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData, DocumentPositionData documentPositionData, SemanticNode parent, Page page) {

        return AtomicTextBlock.builder()
                .id(documentTextData.getId())
@@ -1,14 +1,12 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -50,6 +48,7 @@ public class TablePageBlock extends AbstractPageBlock {
        return getColCount() == 0 || getRowCount() == 0;
    }

+
    public List<List<Cell>> getRows() {

        if (rows == null) {
@@ -276,21 +275,17 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-
    public boolean intersects(Cell cell1, Cell cell2) {
+
        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
            return false;
        }
        double x0 = cell1.getX() + 2;
        double y0 = cell1.getY() + 2;
-        return (cell2.x + cell2.width > x0 &&
-                cell2.y + cell2.height > y0 &&
-                cell2.x < x0 + cell1.getWidth() -2 &&
-                cell2.y < y0 + cell1.getHeight() -2);
+        return (cell2.x + cell2.width > x0 && cell2.y + cell2.height > y0 && cell2.x < x0 + cell1.getWidth() - 2 && cell2.y < y0 + cell1.getHeight() - 2);
    }


-
    @Override
    public String getText() {

@@ -328,8 +323,6 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-
-
    public String getTextAsHtml() {

        StringBuilder sb = new StringBuilder();
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import java.util.ArrayList;
 import java.util.List;
+
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.Getter;
@@ -73,7 +73,7 @@ public class TextPageBlock extends AbstractPageBlock {

        return sequences.get(0).getPageWidth();
    }
-    
+

    public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {

@@ -82,6 +82,7 @@ public class TextPageBlock extends AbstractPageBlock {
        return fromTextPositionSequences(sequences);
    }

+
    public static TextPageBlock fromTextPositionSequences(List<TextPositionSequence> wordBlockList) {

        TextPageBlock textBlock = null;
@@ -133,7 +134,6 @@ public class TextPageBlock extends AbstractPageBlock {
    }


-
    /**
     * Returns the minX value in pdf coordinate system.
     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -9,6 +9,8 @@ import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;

+import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;

@@ -140,7 +142,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minX value
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMinXDirAdj() {

        return textPositions.get(0).getXDirAdj();
@@ -154,7 +157,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxX value
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMaxXDirAdj() {

        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
@@ -168,7 +172,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMinYDirAdj() {

        return textPositions.get(0).getYDirAdj() - getTextHeight();
@@ -182,7 +187,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMaxYDirAdj() {

        return textPositions.get(0).getYDirAdj();
@@ -190,24 +196,32 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getHeight() {

        return getMaxYDirAdj() - getMinYDirAdj();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getWidth() {

        return getMaxXDirAdj() - getMinXDirAdj();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getFont() {

        if (textPositions.get(0).getFontName() == null) {
@@ -217,6 +231,8 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getFontStyle() {

        if (textPositions.get(0).getFontName() == null) {
@@ -236,12 +252,16 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getFontSize() {

        return textPositions.get(0).getFontSizeInPt();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getSpaceWidth() {

        return textPositions.get(0).getWidthOfSpace();
@@ -257,7 +277,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return bounding box of the word in Pdf Coordinate System
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    @SneakyThrows
    public Rectangle getRectangle() {

@@ -9,10 +9,10 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;

 import lombok.RequiredArgsConstructor;

@@ -20,8 +20,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
 public class ImageServiceResponseAdapter {

-
-    public Map<Integer, List<ClassifiedImage>> buildClassifiedImagesPerPage(ImageServiceResponse imageServiceResponse ) {
+    public Map<Integer, List<ClassifiedImage>> buildClassifiedImagesPerPage(ImageServiceResponse imageServiceResponse) {

        Map<Integer, List<ClassifiedImage>> images = new HashMap<>();
        imageServiceResponse.getData().forEach(imageMetadata -> {
@@ -0,0 +1,74 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTable;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTableData;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableData;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class TableExtractorResponseAdapter {
+
+    public Map<Integer, List<TableExtractorCells>> buildExtractedTablesPerPage(TableExtractorResponse tableExtractorResponse) {
+        Map<Integer, List<TableExtractorCells>> tableCells = new HashMap<>();
+        tableExtractorResponse.getData()
+                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_number(), tableCell -> new ArrayList<>())
+                        .addAll(convertTableCells(tableData.getTables())));
+
+        return tableCells;
+
+    }
+
+    public List<TableExtractorCells> convertTableCells(List<ExtractedTable> tableObjects) {
+
+        List<TableExtractorCells> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            TableExtractorCells tableCells = new TableExtractorCells();
+            tableCells.setX0(t.getTable().getBbox().get(0));
+            tableCells.setX1(t.getTable().getBbox().get(2));
+            tableCells.setY0(t.getTable().getBbox().get(1));
+            tableCells.setY1(t.getTable().getBbox().get(3));
+            tableCells.setWidth(tableCells.getX1()- tableCells.getX0());
+            tableCells.setHeight(tableCells.getY1()- tableCells.getY0());
+            tableCells.setLabel(t.getTable().getLabel());
+            log.info("Parsed table cell {} with label {}",tableCells, tableCells.getLabel());
+            parsedTableCells.add(tableCells);
+            t.getObjects().forEach(o -> {
+                TableExtractorCells objectCell = new TableExtractorCells();
+                objectCell.setX0(t.getTable().getBbox().get(0));
+                objectCell.setX1(t.getTable().getBbox().get(2));
+                objectCell.setY0(t.getTable().getBbox().get(1));
+                objectCell.setY1(t.getTable().getBbox().get(3));
+                objectCell.setWidth(objectCell.getX1()- objectCell.getX0());
+                objectCell.setHeight(objectCell.getY1()- objectCell.getY0());
+                objectCell.setLabel(o.getLabel());
+                log.info("Parsed object cell {} with label {}",objectCell, objectCell.getLabel());
+                parsedTableCells.add(objectCell);
+            });
+        });
+
+        return parsedTableCells;
+
+    }
+
+
+}
@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExtractedTable {
+
+    private boolean rotated;
+    private ExtractedTableData table;
+    private List<ExtractedTableData> objects;
+
+}
@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExtractedTableData {
+
+    private String label;
+    private float score;
+    private List<Float> bbox;
+
+}
@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorCells {
+    private float x0;
+    private float y0;
+    private float x1;
+    private float y1;
+    private float width;
+    private float height;
+    private String label;
+
+}
@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorData {
+
+    private int page_number;
+    private int image;
+    private List<ExtractedTable> tables;
+}
@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorResponse {
+
+    private String dossierId;
+    private String fileId;
+    private String targetFileExtension;
+    private String responseFileExtension;
+    private String X_TENANT_ID;
+    private List<TableExtractorData> data;
+
+}
@@ -25,6 +25,7 @@ public class BodyTextFrameService {
    private static final float RULING_HEIGHT_THRESHOLD = 0.15f; // multiplied with page height. Header/Footer Rulings must be within that border of the page.
    private static final float RULING_WIDTH_THRESHOLD = 0.75f; // multiplied with page width. Header/Footer Rulings must be at least that wide.

+
    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {

        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
@@ -155,8 +156,9 @@ public class BodyTextFrameService {
                        continue;
                    }

-                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || MarkedContentUtils.intersects(textBlock,
+                            page.getMarkedContentBboxPerType(),
+                            MarkedContentUtils.FOOTER)) {
                        continue;
                    }

@@ -22,7 +22,6 @@ public class DividingColumnDetectionService {

    public List<Rectangle2D> detectColumns(PageContents pageContents) {

-
        if (pageContents.getSortedTextPositionSequences().size() < 2) {
            return List.of(pageContents.getCropBox());
        }
@@ -72,11 +72,13 @@ public class GapDetectionService {
        return mirrorY(RectangleTransformations.toRectangle2D(textPosition.getRectangle()));
    }

+
    private static Rectangle2D mirrorY(Rectangle2D rectangle2D) {

        return new Rectangle2D.Double(rectangle2D.getX(), Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY()), rectangle2D.getWidth(), Math.abs(rectangle2D.getHeight()));
    }

+
    private static void addGapToLine(Rectangle2D currentTextPosition, Rectangle2D previousTextPosition, XGapsContext context) {

        context.gapsInCurrentLine.add(new Rectangle2D.Double(previousTextPosition.getMaxX(),
@@ -6,7 +6,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
 import java.util.stream.Stream;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;

@@ -51,7 +50,9 @@ public class GapsAcrossLinesService {
        }

        return columnFactory.outputGaps.stream()
-                .filter(gapAcrossLines -> columnFactory.outputGaps.stream().filter(gapAcrossLines::intersectsX).noneMatch(gapAcrossLines1 -> gapAcrossLines1.lineCount > gapAcrossLines.lineCount))
+                .filter(gapAcrossLines -> columnFactory.outputGaps.stream()
+                        .filter(gapAcrossLines::intersectsX)
+                        .noneMatch(gapAcrossLines1 -> gapAcrossLines1.lineCount > gapAcrossLines.lineCount))
                .filter(gapAcrossLines -> Math.abs(gapAcrossLines.rectangle2D.getMinX() - mainBodyTextFrame.getMinX()) > DISTANCE_TO_BORDER_THRESHOLD)
                .filter(gapAcrossLines -> Math.abs(gapAcrossLines.rectangle2D.getMaxX() - mainBodyTextFrame.getMaxX()) > DISTANCE_TO_BORDER_THRESHOLD)
                .map(GapAcrossLines::getRectangle2D)
@@ -6,8 +6,8 @@ import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -16,8 +16,7 @@ public class MainBodyTextFrameExtractionService {

    public Rectangle2D calculateMainBodyTextFrame(LineInformation lineInformation) {

-        Rectangle2D mainBodyTextFrame = lineInformation.getLineBBox().stream()
-                .collect(RectangleTransformations.collectBBox());
+        Rectangle2D mainBodyTextFrame = lineInformation.getLineBBox().stream().collect(RectangleTransformations.collectBBox());

        return RectangleTransformations.pad(mainBodyTextFrame, mainBodyTextFrame.getWidth() * TEXT_FRAME_PAD_WIDTH, mainBodyTextFrame.getHeight() * TEXT_FRAME_PAD_HEIGHT);
    }
@@ -52,7 +52,7 @@ public class PageContentExtractor {
                        stripper.getRulings()));
            }
        }
-    
+
        return textPositionSequencesPerPage;
    }

@@ -5,9 +5,9 @@ import java.util.List;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;

@Service
 public class SimplifiedSectionTextService {
@@ -23,4 +23,5 @@ public class SimplifiedSectionTextService {

        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
    }
+
 }
@@ -1,9 +1,20 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;

-
 // TODO: figure out, why this fails the build
 // import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;

+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
@@ -11,12 +22,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;

@Service
@SuppressWarnings("all")
@@ -83,13 +88,13 @@ public class TaasBlockificationService {
                continue;
            }

-
            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
            boolean isListIdentifier = listIdentifierPattern.find();

            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;

-            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
+            boolean sameFont = previousTextBlock.getMostPopularWordFont()
+                    .equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
 //            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;

            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
@@ -119,8 +124,9 @@ public class TaasBlockificationService {
            }
            alreadyMerged.add(textPageBlock);
            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
-                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
-                    .toList());
+                    textPageBlocks.stream()
+                            .filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2))
+                            .peek(alreadyMerged::add)).toList());
        }
        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
    }
@@ -163,8 +169,7 @@ public class TaasBlockificationService {
        while (itty.hasNext()) {
            TextPageBlock block = (TextPageBlock) itty.next();

-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(
-                    block.getMaxY(),
+            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
                previous.add(block);
@@ -189,7 +194,6 @@ public class TaasBlockificationService {
        TextPositionSequence prev = null;
        // TODO: make static final constant

-
        boolean wasSplitted = false;
        Float splitX1 = null;
        for (TextPositionSequence word : textPositions) {
@@ -5,7 +5,6 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -13,6 +12,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -63,16 +63,16 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
-        ) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
-        ) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -11,6 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RedactManagerClassificationService {

-
    public void classifyDocument(ClassificationDocument document) {

        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
@@ -52,14 +51,16 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -12,6 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -27,7 +27,6 @@ public class TaasClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
@@ -57,11 +56,13 @@ public class TaasClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation())) {
            textBlock.setClassification(PageBlockType.HEADER);
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -18,8 +18,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
@@ -31,6 +29,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

@@ -8,10 +8,10 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Objects;

+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;

 import lombok.experimental.UtilityClass;

@@ -110,6 +110,7 @@ public class SearchTextWithTextPositionFactory {
        return context.stringIdx - context.lastHyphenIdx < MAX_HYPHEN_LINEBREAK_DISTANCE;
    }

+
    private static List<Boundary> mergeToBoundaries(List<Integer> integers) {

        if (integers.isEmpty()) {
@@ -125,8 +126,9 @@ public class SearchTextWithTextPositionFactory {
            }
            end = current + 1;
        }
-        if (boundaries.isEmpty())
+        if (boundaries.isEmpty()) {
            boundaries.add(new Boundary(start, end));
+        }
        return boundaries;
    }

@@ -138,6 +140,7 @@ public class SearchTextWithTextPositionFactory {
        }
    }

+
    private boolean isLineBreak(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition) {

        return Objects.equals(currentTextPosition.getUnicode(), "\n") || isDeltaYLargerThanTextHeight(currentTextPosition, previousTextPosition);
@@ -163,17 +166,7 @@ public class SearchTextWithTextPositionFactory {

    private boolean isHyphen(String unicodeCharacter) {

-        return Objects.equals(unicodeCharacter, "-") || //
-                Objects.equals(unicodeCharacter, "~") || //
-                Objects.equals(unicodeCharacter, "‐") || //
-                Objects.equals(unicodeCharacter, "‒") || //
-                Objects.equals(unicodeCharacter, "⁻") || //
-                Objects.equals(unicodeCharacter, "−") || //
-                Objects.equals(unicodeCharacter, "﹣") || //
-                Objects.equals(unicodeCharacter, "゠") || //
-                Objects.equals(unicodeCharacter, "⁓") || //
-                Objects.equals(unicodeCharacter, "‑") || //
-                Objects.equals(unicodeCharacter, "\u00AD");
+        return false;
    }


@@ -11,12 +11,12 @@ import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TableMergingUtility;

 import lombok.experimental.UtilityClass;
@@ -8,15 +8,15 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import lombok.experimental.UtilityClass;
@@ -2,10 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;

 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -7,11 +7,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -9,7 +8,6 @@ import java.util.Map;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;

@@ -329,6 +329,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
    }

+
    @Override
    public String getText(PDDocument doc) throws IOException {

@@ -25,10 +25,23 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.regex.Pattern;

-import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.pdfbox.cos.COSDictionary;
@@ -46,6 +59,8 @@ import org.apache.pdfbox.text.TextPositionComparator;

 import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;

+import lombok.Getter;
+
 /**
 * This is just a copy except i only adjusted lines 594-607 cause this is a bug in Pdfbox.
 * see S416.pdf
@@ -194,40 +209,33 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    }


+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {

-    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
-    {
        PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
-        if (this.currentMarkedContents.isEmpty())
-        {
+        if (this.currentMarkedContents.isEmpty()) {
            this.markedContents.add(markedContent);
-        }
-        else
-        {
-            PDMarkedContent currentMarkedContent =
-                    this.currentMarkedContents.peek();
-            if (currentMarkedContent != null)
-            {
+        } else {
+            PDMarkedContent currentMarkedContent = this.currentMarkedContents.peek();
+            if (currentMarkedContent != null) {
                currentMarkedContent.addMarkedContent(markedContent);
            }
        }
        this.currentMarkedContents.push(markedContent);
    }

+
    @Override
-    public void endMarkedContentSequence()
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void endMarkedContentSequence() {
+
+        if (!this.currentMarkedContents.isEmpty()) {
            this.currentMarkedContents.pop();
        }
    }


-    public void xobject(PDXObject xobject)
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void xobject(PDXObject xobject) {
+
+        if (!this.currentMarkedContents.isEmpty()) {
            this.currentMarkedContents.peek().addXObject(xobject);
        }
    }
@@ -635,7 +643,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                        var normalized = normalize(line);
 //                        normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent()

-
                        lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine);
                        writeLine(normalized, current.isParagraphStart);
                        line.clear();
@@ -914,8 +921,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    textList.add(text);
                }
            }
-            if (!this.currentMarkedContents.isEmpty())
-            {
+            if (!this.currentMarkedContents.isEmpty()) {
                this.currentMarkedContents.peek().addText(text);
            }
        }
@@ -2102,7 +2108,9 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            return endParagraphWritten;
        }

-        public void setEndParagraphWritten(){
+
+        public void setEndParagraphWritten() {
+
            endParagraphWritten = true;
        }

@@ -2145,7 +2153,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            this.isHangingIndent = true;
        }

-
    }

 }
@@ -1,14 +1,16 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.visualization;

+import java.awt.Color;
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
+import java.io.OutputStream;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;

-import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.cos.COSDictionary;
 import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
@@ -31,10 +33,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.visualization.
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.LayoutGrid;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;

-import io.micrometer.observation.Observation;
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -49,26 +49,28 @@ public class ViewerDocumentService {
    public static final float LINE_WIDTH = 1f;

    private final LayoutGridService layoutGridService;
-    private final ObservationRegistry observationRegistry;


    @SneakyThrows
-    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
-    public void createViewerDocument(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {
+    public void createViewerDocument(PDDocument pdDocument,
+                                     Document document,
+                                     OutputStream outputStream,
+                                     Map<Integer,List<TableExtractorCells>> extractedTableCells,
+                                     boolean layerVisibilityDefaultValue) {

-        Path tmpFile = Files.createTempFile("tmpViewerDocument", ".pdf");
-        PDDocument pdDocument = openPDDocument(originFile);
        LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
-
-        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
+        // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
+        // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
+        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
+        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
+        PDOptionalContentGroup visualLayoutParsingLayer = addLayerToDocument(pdDocument, dictionariesToUpdate, true);
        PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);

        for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
-
            PDPage pdPage = pdDocument.getPage(pageNumber);
-//
+
            AffineTransform textDeRotationMatrix = getTextDeRotationTransform(pdPage);
-            addLayerToPageResources(pdPage);
+            addLayerToPageRessources(pdPage);

            // We need to save the graphics state before, such that our appended content cannot be affected by previous content streams with side effects,
            // e.g. not escaped matrix transformations.
@@ -78,91 +80,40 @@ public class ViewerDocumentService {
            assert pageNumber == visualizationsOnPage.getPageNumber();
            // We need to append to the content stream, otherwise the content could be overlapped by following content.
            try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true)) {
-
-                contentStream.beginMarkedContent(COSName.OC, layer);
+                contentStream.beginMarkedContent(COSName.OC, visualLayoutParsingLayer);
                contentStream.saveGraphicsState();

                contentStream.setLineWidth(LINE_WIDTH);
-                for (ColoredLine coloredLine : visualizationsOnPage.getColoredLines()) {
-                    contentStream.setStrokingColor(coloredLine.color());
-                    contentStream.moveTo((float) coloredLine.line().getX1(), (float) coloredLine.line().getY1());
-                    contentStream.lineTo((float) coloredLine.line().getX2(), (float) coloredLine.line().getY2());
+                for (TableExtractorCells tableCells : extractedTableCells.get(pageNumber)) {
+                    contentStream.setStrokingColor(new Color(0xB700FF));
+                    contentStream.addRect((float) tableCells.getX0(), (float) tableCells.getY0(), (float) tableCells.getWidth(), (float) tableCells.getHeight());
                    contentStream.stroke();
-                }
-                for (ColoredRectangle coloredRectangle : visualizationsOnPage.getColoredRectangles()) {
-                    contentStream.setStrokingColor(coloredRectangle.color());
-                    Rectangle2D r = coloredRectangle.rectangle2D();
-                    contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
-                    contentStream.stroke();
-                }
-                for (FilledRectangle filledRectangle : visualizationsOnPage.getFilledRectangles()) {
-                    contentStream.setNonStrokingColor(filledRectangle.color());
-                    PDExtendedGraphicsState graphicsState = new PDExtendedGraphicsState();
-                    graphicsState.setNonStrokingAlphaConstant(filledRectangle.alpha());
-                    contentStream.setGraphicsStateParameters(graphicsState);
-                    Rectangle2D r = filledRectangle.rectangle2D();
-                    contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
-                    contentStream.fill();
-                }
-                for (PlacedText placedText : visualizationsOnPage.getPlacedTexts()) {
                    contentStream.setFont(font, FONT_SIZE);
                    contentStream.beginText();
                    Matrix textMatrix = new Matrix((float) textDeRotationMatrix.getScaleX(),
                            (float) textDeRotationMatrix.getShearX(),
                            (float) textDeRotationMatrix.getShearY(),
                            (float) textDeRotationMatrix.getScaleY(),
-                            (float) placedText.lineStart().getX(),
-                            (float) placedText.lineStart().getY());
-                    textMatrix.translate(-((font.getStringWidth(placedText.text()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
+                            tableCells.getX0() ,
+                            tableCells.getY0());
+                    textMatrix.translate(-((font.getStringWidth(tableCells.getLabel()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
                    contentStream.setTextMatrix(textMatrix);
-                    contentStream.showText(placedText.text());
+                    contentStream.showText(tableCells.getLabel());
                    contentStream.endText();
                }
                contentStream.restoreGraphicsState();
                contentStream.endMarkedContent();
            }
-
-            if (pageNumber % 500 == 0 && pageNumber != 0) { // re-open document every once in a while to save on RAM
-                log.info("Incremental save after {} pages", pageNumber);
-                observedIncrementalSave(pdDocument, destinationFile);
-                pdDocument.close();
-                Files.copy(destinationFile.toPath(), tmpFile, StandardCopyOption.REPLACE_EXISTING);
-                pdDocument = openPDDocument(tmpFile.toFile());
-                layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
-            }
-
+            dictionariesToUpdate.add(pdPage.getCOSObject());
+            dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
        }
-        observedIncrementalSave(pdDocument, destinationFile);
-
-        tmpFile.toFile().delete();
-        pdDocument.close();
+        dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
+//        dictionariesToUpdate.add(pdDocument.getDocument().getTrailer());
+        pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
    }


-    private static PDDocument openPDDocument(File tmpFile) throws IOException {
-
-        PDDocument pdDocument;
-        pdDocument = Loader.loadPDF(tmpFile);
-        pdDocument.setAllSecurityToBeRemoved(true);
-        return pdDocument;
-    }
-
-
-    @SneakyThrows
-    private void observedIncrementalSave(PDDocument pdDocument, File outputFile) {
-
-        Observation.createNotStarted("ViewerDocumentService", observationRegistry).contextualName("incremental-save").observe(() -> {
-            try {
-                pdDocument.save(outputFile);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        });
-
-    }
-
-
-    private static void addLayerToPageResources(PDPage pdPage) {
+    private static void addLayerToPageRessources(PDPage pdPage) {

        PDResources resources = pdPage.getResources();
        if (resources == null) {
@@ -183,7 +134,7 @@ public class ViewerDocumentService {
    }


-    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, boolean layerVisibilityDefaultValue) {
+    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate, boolean layerVisibilityDefaultValue) {

        PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
        PDOptionalContentProperties ocprops = catalog.getOCProperties();
@@ -199,7 +150,7 @@ public class ViewerDocumentService {
            ocprops.addGroup(layer);
        }
        ocprops.setGroupEnabled(layer, layerVisibilityDefaultValue);
-//        dictionariesToUpdate.add(catalog.getCOSObject());
+        dictionariesToUpdate.add(catalog.getCOSObject());
        return layer;
    }

@@ -1,12 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import lombok.experimental.UtilityClass;
-import org.apache.pdfbox.cos.COSName;
-import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
-import org.apache.pdfbox.text.TextPosition;
-
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.Collections;
@@ -14,12 +7,22 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;

+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.text.TextPosition;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.experimental.UtilityClass;
+
@UtilityClass
 public class MarkedContentUtils {

    public static final String HEADER = "Header";
    public static final String FOOTER = "Footer";

+
    public List<Rectangle2D> getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {

        if (markedContents == null) {
@@ -31,7 +34,8 @@ public class MarkedContentUtils {
                .filter(m -> m.getProperties() != null)
                .filter(m -> m.getProperties().getItem("Subtype") != null)
                .filter(m -> ((COSName) m.getProperties().getItem("Subtype")).getName().equals(subtype))
-                .map(PDMarkedContent::getContents).flatMap(Collection::stream)
+                .map(PDMarkedContent::getContents)
+                .flatMap(Collection::stream)
                .filter(t -> t instanceof TextPosition)
                .map(t -> (TextPosition) t)
                .filter(t -> !t.getUnicode().equals(" "))
@@ -41,16 +45,19 @@ public class MarkedContentUtils {
            return Collections.emptyList();
        }

-        return markedContentByYPosition.values().stream()
-                .map(textPositions -> new TextPositionSequence(textPositions.stream()
-                        .toList(), 0, true)
-                        .getRectangle())
-                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList());
+        return markedContentByYPosition.values()
+                .stream()
+                .map(textPositions -> new TextPositionSequence(textPositions.stream().toList(), 0, true).getRectangle())
+                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight())))
+                .collect(Collectors.toList());
    }


    public boolean intersects(TextPageBlock textBlock, Map<String, List<Rectangle2D>> markedContentBboxPerType, String type) {
-        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).stream().anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
+
+        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type)
+                .stream()
+                .anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
    }

 }
@@ -19,10 +19,9 @@ public final class PositionUtils {

        double threshold = textBlock.getMostPopularWordHeight() * 3;

-        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX()
-                && textBlock.getPdfMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth()
-                && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY()
-                && textBlock.getPdfMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
+        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX() && textBlock.getPdfMaxX() - threshold < btf.getTopLeft()
+                .getX() + btf.getWidth() && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY() && textBlock.getPdfMaxY() - threshold < btf.getTopLeft()
+                .getY() + btf.getHeight()) {
            return true;
        } else {
            return false;
@@ -41,11 +41,14 @@ public class RectangleTransformations {

        return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DBBoxCollector());
    }
+
+
    public static Collector<Rectangle2D, Rectangle2DBBoxCollector.BBox, Rectangle2D> collectBBox() {

        return new Rectangle2DBBoxCollector();
    }

+
    public static PDRectangle toPDRectangleBBox(List<Rectangle> rectangles) {

        Rectangle2D rectangle2D = RectangleTransformations.rectangleBBox(rectangles);
@@ -70,6 +73,7 @@ public class RectangleTransformations {
        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
    }

+
    public static Rectangle2D rectangleBBox(List<Rectangle> rectangles) {

        return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DBBoxCollector());
@@ -84,6 +88,7 @@ public class RectangleTransformations {
                -redactionLogRectangle.getHeight());
    }

+
    public static Rectangle2D toRectangle2D(PDRectangle rectangle) {

        return new Rectangle2D.Double(rectangle.getLowerLeftX(), rectangle.getLowerLeftY(), rectangle.getWidth(), rectangle.getHeight());
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 import java.util.List;
 import java.util.stream.Collectors;

-
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

@@ -28,15 +28,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPosit
 *
 * @author Ben Litchfield
 */
-public class TextPositionSequenceComparator implements Comparator<TextPositionSequence>
-{
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
+
    @Override
-    public int compare(TextPositionSequence pos1, TextPositionSequence pos2)
-    {
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
        // only compare text that is in the same direction
        int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
-        if (cmp1 != 0)
-        {
+        if (cmp1 != 0) {
            return cmp1;
        }

@@ -54,19 +52,13 @@ public class TextPositionSequenceComparator implements Comparator<TextPositionSe
        float yDifference = Math.abs(pos1YBottom - pos2YBottom);

        // we will do a simple tolerance comparison
-        if (yDifference < .1 ||
-                pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
-                pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
-        {
+        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
            return Float.compare(x1, x2);
-        }
-        else if (pos1YBottom < pos2YBottom)
-        {
+        } else if (pos1YBottom < pos2YBottom) {
            return -1;
-        }
-        else
-        {
+        } else {
            return 1;
        }
    }
+
 }
@@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
    application
    id("com.knecon.fforesight.java-conventions")
-    id("org.springframework.boot") version "3.1.5"
+    id("org.springframework.boot") version "3.1.3"
    id("io.spring.dependency-management") version "1.1.3"
    id("org.sonarqube") version "4.3.0.3225"
    id("io.freefair.lombok") version "8.2.2"
@@ -12,43 +12,31 @@ plugins {

 description = "layoutparser-service-server"

-configurations {
-    all {
-        exclude(group = "commons-logging", module = "commons-logging")
-        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
-        exclude(group = "com.iqser.red.commons", module = "logging-commons")
-    }
-}
-
-
 val jacksonVersion = "2.15.2"
 val pdfBoxVersion = "3.0.0"
-val springBootStarterVersion = "3.1.5"

 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
-    implementation("com.knecon.fforesight:tracing-commons:0.3.0")
-    implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
+    implementation("com.knecon.fforesight:tenant-commons:0.14.0")
+    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
    implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
-    implementation("net.logstash.logback:logstash-logback-encoder:7.4")

    // for integration testing only
    testImplementation(project(":layoutparser-service-internal-api"))

-    testImplementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
-    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0")
    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.3")
 }

 /*
@@ -5,7 +5,6 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
@@ -14,9 +13,6 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingService
 import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.aop.ObservedAspect;
-
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
@@ -27,11 +23,4 @@ public class Application {
        SpringApplication.run(Application.class, args);
    }

-
-    @Bean
-    public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
-
-        return new ObservedAspect(observationRegistry);
-    }
-
 }
@@ -24,7 +24,8 @@ public class MessagingConfiguration {
    @Bean
    public Queue layoutparsingResponseQueue() {

-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
    }


@@ -5,19 +5,10 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/

 fforesight.tenants.remote: true

-logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
-
-logging.type: ${LOGGING_TYPE:CONSOLE}
-kubernetes.namespace: ${NAMESPACE:default}
-project.version: 1.0-SNAPSHOT
-
-
 server:
  port: 8080

 spring:
-  application:
-    name: layoutparser-service
  main:
    allow-circular-references: true # FIXME
  rabbitmq:
@@ -41,13 +32,6 @@ management:
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
  endpoints.web.exposure.include: prometheus, health
-  tracing:
-    enabled: ${TRACING_ENABLED:false}
-    sampling:
-      probability: ${TRACING_PROBABILITY:1.0}
-  otlp:
-    tracing:
-      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}


 storage:
@@ -1,17 +0,0 @@
-<configuration>
-
-    <springProperty scope="configuration" name="logType" source="logging.type"/>
-    <springProperty scope="context" name="application.name" source="spring.application.name"/>
-    <springProperty scope="context" name="version" source="project.version"/>
-    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
-    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
-
-    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
-    </appender>
-
-    <root level="INFO">
-        <appender-ref ref="${logType}"/>
-    </root>
-
-</configuration>
@@ -26,6 +26,7 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
@@ -46,11 +47,12 @@ public class BdrJsonBuildTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(File file) {

-        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
-                file,
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                file.toString()));
+        try (PDDocument pdDocument = Loader.loadPDF(file)) {
+            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                    pdDocument,
+                    new ImageServiceResponse(),
+                    new TableServiceResponse(), new TableExtractorResponse()));
+        }
    }


@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

+import org.apache.pdfbox.Loader;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -32,6 +33,7 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
@@ -95,10 +97,9 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                pdfFileResource.getFile(),
+                Loader.loadPDF(pdfFileResource.getFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filePath));
+                new TableServiceResponse(), new TableExtractorResponse()));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -25,8 +25,8 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
    @SneakyThrows
    public void testLayoutParserEndToEnd() {

-        prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        prepareStorage("files/bdr/btd_19_053_1905391.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.TAAS);
        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
    }
@@ -6,7 +6,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;

-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -18,9 +17,7 @@ import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest
 import lombok.SneakyThrows;

 public class DocumentDataTests extends BuildDocumentTest {
-
    @Test
-    @Disabled // This test takes waaaaaay too long, it's ridiculous
    @SneakyThrows
    public void createDocumentDataForAllFiles() {

@@ -39,12 +36,11 @@ public class DocumentDataTests extends BuildDocumentTest {
        for (String pdfFileName : pdfFileNames) {
            System.out.println(pdfFileName);
            DocumentData documentData = DocumentDataMapper.toDocumentData(buildGraph(resource.getFile().toPath().getParent().relativize(Path.of(pdfFileName)).toString()));
-            File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
+                        File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
            outputFile.toPath().getParent().toFile().mkdirs();
            try (var out = new FileOutputStream(outputFile.toString().replace(".pdf", ".json"))) {
                ObjectMapperFactory.create().writeValue(out, documentData);
            }
        }
    }
-
 }
@@ -5,6 +5,7 @@ import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;

+import org.apache.pdfbox.Loader;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;
@@ -15,6 +16,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
@@ -55,10 +57,9 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
    private void writeJsons(Path filename) {

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
+                Loader.loadPDF(filename.toFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
+                new TableServiceResponse(), new TableExtractorResponse()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -1,32 +1,156 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

-import java.io.File;
-import java.nio.file.Path;
+import static com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw.drawRectangle2DList;

+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
+import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Rectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTable;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.GapDetectionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.GapsAcrossLinesService;
+import com.knecon.fforesight.service.layoutparser.processor.services.LineDetectionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;

 public class ViewerDocumentTest extends BuildDocumentTest {

+    @Autowired
+    private SectionsBuilderService sectionsBuilderService;
+
+    @Autowired
+    private RedactManagerClassificationService redactManagerClassificationService;
+
+    @Autowired
+    private ObjectMapper objectMapper;
+
    @Test
    @SneakyThrows
    public void testViewerDocument() {

-        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
+        String fileName = "files/bdr/notMergedParagraphs.pdf";
        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
        LayoutGridService layoutGridService = new LayoutGridService();
-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
-        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
-        var documentFile = new ClassPathResource(fileName).getFile();
-        viewerDocumentService.createViewerDocument(documentFile, document, new File(tmpFileName), true);
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
+        Document document = buildGraph(fileName, LayoutParsingType.TAAS);
+        try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getFile()); var out = new FileOutputStream(tmpFileName)) {
+            viewerDocumentService.createViewerDocument(pdDocument, document, out, null,true);
+        }
+    }
+
+    @Test
+    @SneakyThrows
+    public void testLayoutParsingServiceResults() {
+        String tableSourceFileName ="C:\\Users\\YannikHampe\\Downloads\\b28d9a22b674906813f12b86dda33202.EXTRACTED_TABLES.json\\b28d9a22b674906813f12b86dda33202.EXTRACTED_TABLES.json";
+        Path pdfFileResource = Path.of("C:\\Users\\YannikHampe\\Downloads\\2009-1048395_50pages_tables.pdf");
+        String tmpFileName = "C:/Users/YANNIK~1/AppData/Local/Temp/rectangles."+pdfFileResource.getFileName();
+        PDDocument pdDocument = Loader.loadPDF(pdfFileResource.toFile());
+
+        try (InputStream inputStream = Files.newInputStream(Path.of(tableSourceFileName))) {
+            TableExtractorResponse tableExtractorResponse = objectMapper.readValue(inputStream, TableExtractorResponse.class);
+            tableExtractorResponse.getData().forEach(data -> {
+                List<TableCells> tableCells = convertTableCells(data.getTables());
+            });
+            inputStream.close();
+        }
+        try (var out = new FileOutputStream(tmpFileName)) {
+            pdDocument.save(out);
+            pdDocument.close();
+        }
+
+    }
+
+    public List<TableCells> convertTableCells(List<ExtractedTable> tableObjects) {
+
+        List<TableCells> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            System.out.println(t.getTable().getLabel());
+            TableCells tableCells = new TableCells();
+            tableCells.setX0(t.getTable().getBbox().get(0));
+            tableCells.setX1(t.getTable().getBbox().get(2));
+            tableCells.setY0(t.getTable().getBbox().get(1));
+            tableCells.setY1(t.getTable().getBbox().get(3));
+            tableCells.setWidth(tableCells.getX1()- tableCells.getX0());
+            tableCells.setHeight(tableCells.getY1()- tableCells.getY0());
+            parsedTableCells.add(tableCells);
+            t.getObjects().forEach(o -> {
+                System.out.println(o.getLabel());
+                TableCells objectCell = new TableCells();
+                objectCell.setX0(t.getTable().getBbox().get(0));
+                objectCell.setX1(t.getTable().getBbox().get(2));
+                objectCell.setY0(t.getTable().getBbox().get(1));
+                objectCell.setY1(t.getTable().getBbox().get(3));
+                objectCell.setWidth(objectCell.getX1()- objectCell.getX0());
+                objectCell.setHeight(objectCell.getY1()- objectCell.getY0());
+                parsedTableCells.add(objectCell);
+            });
+        });
+
+        return parsedTableCells;
+
+    }
+
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                originDocument,
+                new ImageServiceResponse(),
+                new TableServiceResponse(), new TableExtractorResponse());
+
+        redactManagerClassificationService.classifyDocument(classificationDocument);
+
+        sectionsBuilderService.buildSections(classificationDocument);
+
+        return classificationDocument;
    }

 }
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.server.segmentation;
 import static org.assertj.core.api.Assertions.assertThat;

 import java.awt.geom.Rectangle2D;
-import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Path;
@@ -16,6 +15,8 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;

+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -32,6 +33,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePag
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
@@ -61,13 +63,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {


    @SneakyThrows
-    public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument, TableServiceResponse tableServiceResponse) {

        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                originDocument,
                new ImageServiceResponse(),
-                tableServiceResponse,
-                "document");
+                tableServiceResponse, new TableExtractorResponse());

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -78,7 +79,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {


    @SneakyThrows
-    public ClassificationDocument buildClassificationDocument(File originDocument) {
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {

        return buildClassificationDocument(originDocument, new TableServiceResponse());
    }
@@ -89,7 +90,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");

@@ -103,7 +104,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()), tableServiceResponse);

        toHtml(document, "/tmp/ScanRotationBorder.html");
    }
@@ -116,7 +117,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()), tableServiceResponse);
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();

@@ -156,7 +157,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(table.getColCount()).isEqualTo(6);
@@ -170,7 +171,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -188,7 +189,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(9);
@@ -206,7 +207,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -224,7 +225,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

@@ -241,7 +242,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/211.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

@@ -258,7 +259,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -299,7 +300,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        toHtml(document, "/tmp/html.html");

@@ -319,7 +320,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -332,7 +333,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185_fixed.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -345,7 +346,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izZRMS (CZ) fRR Part B7_Page123.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 6);

@@ -364,7 +365,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04_Page11.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 3);

@@ -380,7 +381,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10_Page532.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 9, 0, 0);
@@ -393,7 +394,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -407,7 +408,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 6, 7, 0);
@@ -420,7 +421,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
        validateTable(document, 0, 10, 6, 0, 0);
@@ -433,7 +434,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);
        validateTable(document, 0, 2, 2, 0, 0);
@@ -448,7 +449,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -464,7 +465,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -479,7 +480,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -494,7 +495,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -509,7 +510,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

@@ -526,7 +527,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -541,7 +542,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -556,7 +557,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -570,7 +571,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -20,6 +20,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
@@ -58,7 +59,7 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
    public void testTableExtraction() {

        LayoutGridService layoutGridService = new LayoutGridService();
-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);

        ClassPathResource resource = new ClassPathResource("files");
        List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
@@ -77,15 +78,13 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
    private void writeJsons(Path filename) {

        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
+                Loader.loadPDF(filename.toFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
+                new TableServiceResponse(), new TableExtractorResponse()));
        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
+                Loader.loadPDF(filename.toFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
+                new TableServiceResponse(), new TableExtractorResponse()));
        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
@@ -1,26 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

-import java.io.InputStream;
-import java.util.Map;
-import java.util.Optional;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
@@ -30,8 +9,24 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorage
 import com.knecon.fforesight.service.layoutparser.server.Application;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
-
 import lombok.SneakyThrows;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.*;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import java.io.InputStream;
+import java.util.Optional;
+
+import javax.swing.text.html.Option;

@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -53,6 +48,8 @@ public abstract class AbstractTest {
    protected final static String ORIGIN_FILE_ID = "origin";
    protected final static String TABLE_FILE_ID = "table";
    protected final static String IMAGE_FILE_ID = "image";
+
+    protected final static String TABLE_EXTRACTOR_FILE_ID = "extractedTable";
    protected final static String STRUCTURE_FILE_ID = "structure";
    protected final static String TEXT_FILE_ID = "texts";
    protected final static String POSITION_FILE_ID = "positions";
@@ -69,6 +66,7 @@ public abstract class AbstractTest {
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .tableExtractorFileId(Optional.of(TABLE_EXTRACTOR_FILE_ID))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
                .positionBlockFileStorageId(POSITION_FILE_ID)
@@ -96,7 +94,7 @@ public abstract class AbstractTest {
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(String file) {

-        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json");
+        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json","table_extractor_response/empty.json");
    }


@@ -107,15 +105,14 @@ public abstract class AbstractTest {
        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
    }

-
    protected LayoutParsingRequest buildDefaultLayoutParsingRequest(LayoutParsingType layoutParsingType) {

        return LayoutParsingRequest.builder()
-                .identifier(Map.of("fileId", "1337"))
                .layoutParsingType(layoutParsingType)
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .tableExtractorFileId(Optional.of(TABLE_EXTRACTOR_FILE_ID))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
                .positionBlockFileStorageId(POSITION_FILE_ID)
@@ -125,23 +122,24 @@ public abstract class AbstractTest {
                .build();
    }

-
    @SneakyThrows
-    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile) {
+    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile, String tableExtractorResponseFile) {

        ClassPathResource pdfFileResource = new ClassPathResource(file);
        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
        ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
+        ClassPathResource tableExtractorResponseFileRessource = new ClassPathResource(tableExtractorResponseFile);

-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream());
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream(),tableExtractorResponseFileRessource.getInputStream());
    }


    @SneakyThrows
-    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream) {
+    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream, InputStream tableExtractorResponseFileStream) {

        storageService.storeObject(TenantContext.getTenantId(), IMAGE_FILE_ID, imageInfoStream);
        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(), TABLE_EXTRACTOR_FILE_ID, tableExtractorResponseFileStream);
        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);

        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
@@ -162,6 +160,7 @@ public abstract class AbstractTest {
    @ComponentScan("com.knecon.fforesight.service.layoutparser")
    public static class TestConfiguration {

+
        @Bean
        @Primary
        public StorageService inmemoryStorage() {
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

-import java.io.File;
+import java.io.InputStream;

+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

@@ -9,6 +11,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;

@@ -23,9 +26,11 @@ public abstract class BuildDocumentTest extends AbstractTest {
    @SneakyThrows
    protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {

-        File fileResource = new ClassPathResource(filename).getFile();
+        ClassPathResource fileResource = new ClassPathResource(filename);
        prepareStorage(filename);
-        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), filename);
+        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream.readAllBytes())) {
+            return layoutParsingPipeline.parseLayout(layoutParsingType, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), new TableExtractorResponse());
+        }
    }


@@ -40,7 +45,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
    protected Document buildGraph(String filename, LayoutParsingType layoutParsingType) {

        if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
-            prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json");
+            prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json","table_extractor_response/empty.json");
        } else {
            prepareStorage(filename);
        }
@@ -7,9 +7,6 @@ fforesight.tenants.remote: true
 server:
  port: 8080

-logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
-logging.type: ${LOGGING_TYPE:CONSOLE}
-
 spring:
  main:
    allow-circular-references: true # FIXME
@@ -0,0 +1,16 @@
+<Configuration>
+
+    <Appenders>
+        <Console name="CONSOLE" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="warn">
+            <AppenderRef ref="CONSOLE"/>
+        </Root>
+        <Logger name="com.iqser" level="info"/>
+    </Loggers>
+
+</Configuration>
@@ -1,17 +0,0 @@
-<configuration>
-
-    <springProperty scope="configuration" name="logType" source="logging.type"/>
-    <springProperty scope="context" name="application.name" source="spring.application.name"/>
-    <springProperty scope="context" name="version" source="project.version"/>
-    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
-    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
-
-    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
-    </appender>
-
-    <root level="INFO">
-        <appender-ref ref="${logType}"/>
-    </root>
-
-</configuration>
@@ -0,0 +1,8 @@
+{
+  "dossierId": "123",
+  "fileId": "123",
+  "operation": "table",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "TABLES.json.gz",
+  "extractedTableData": []
+}
@@ -2,14 +2,7 @@
 dir=${PWD##*/}
 gradle assemble

-# Get the current Git branch
-branch=$(git rev-parse --abbrev-ref HEAD)
+buildNumber=${1:-1}

-# Get the short commit hash (first 5 characters)
-commit_hash=$(git rev-parse --short=5 HEAD)
-
-# Combine branch and commit hash
-buildName="${USER}-${branch}-${commit_hash}"
-
-gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName --no-build-cache
-echo "nexus.knecon.com:5001/ff/${dir}-service-server:$buildName"
+gradle bootBuildImage --cleanCache --publishImage -Pversion=layout-parser-yannik-$buildNumber --stacktrace
+echo "nexus.knecon.com:5001/red/${dir}-server-v1:table-extractor-yannik-$buildNumber"
Author	SHA1	Message	Date
yhampe	aa1e77dda3	RED-7375 table extractor prototype	2024-02-01 08:23:53 +01:00
yhampe	0c3b910088	RED-7375 table extractor prototype	2024-01-30 11:59:18 +01:00
yhampe	b4e5f2da2f	RED-7375 table extractor prototype added label changed colour of bounding boxes	2024-01-17 16:58:53 +01:00
yhampe	b47b187c8a	RED-7375: table extractor prototype added layer for table extractor results in ViewerDocumentService	2024-01-15 09:43:23 +01:00
yhampe	0d4800622d	RED-7375:Integrate Table Parsing Service * integrated table parsing service * for visualization simpy drawing the extracted table in ViewerDocumentService	2023-12-01 13:26:16 +01:00
yhampe	cc492bc50d	RED-7375:Integrate Table Parsing Service * fixed merging mistake	2023-11-29 10:06:05 +01:00
yhampe	1c3c5385e1	Merge remote-tracking branch 'origin/RED-7375' into RED-7375	2023-11-29 09:40:24 +01:00
yhampe	bce2558133	RED-7375:Integrate Table Parsing Service * fixed end2end test by introducing sample table extractor response	2023-11-29 09:39:58 +01:00
yhampe	2200574b6d	RED-7375:Integrate Table Parsing Service * disabled not needed test	2023-11-29 09:39:58 +01:00
yhampe	775c943f7e	RED-7375: * using ViewerDocumentService to draw TableExtractorResponse into documents	2023-11-29 09:39:53 +01:00
yhampe	b7fe6fd3c4	RED-7375:Integrate Table Parsing Service * fixed end2end test by introducing sample table extractor response	2023-11-29 09:37:47 +01:00
yhampe	3dd447ebef	RED-7375:Integrate Table Parsing Service * disabled not needed test	2023-11-29 08:41:31 +01:00
yhampe	f4e93ef03b	RED-7375: * using ViewerDocumentService to draw TableExtractorResponse into documents	2023-11-28 15:14:04 +01:00