Merge branch 'RED-1137' into 'main'

RED-1137: Do not observe actuator endpoints See merge request fforesight/layout-parser!92
RED-1137: Do not observe actuator endpoints
2023-12-20 14:11:31 +01:00 · 2023-12-20 14:05:00 +01:00 · 2023-12-20 12:40:00 +01:00 · 2023-12-20 12:40:00 +01:00 · 2023-12-13 16:05:01 +01:00 · 2023-12-13 15:31:26 +01:00
65 changed files with 1267 additions and 704 deletions
@@ -10,7 +10,7 @@ deploy:
  script:
    - echo "Building with gradle version ${BUILDVERSION}"
    - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
  artifacts:
    reports:
@@ -24,6 +24,8 @@ tasks.named<Test>("test") {
    reports {
        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
    }
+    minHeapSize = "512m"
+    maxHeapSize = "2048m"
 }

 tasks.test {
@@ -36,9 +36,6 @@ public record LayoutParsingRequest(
        @Schema(description = "Path where the Simplified Text File will be stored.")//
        @NonNull String simplifiedTextStorageId,//
        @Schema(description = "Path where the Viewer Document PDF will be stored.")//
-        @NonNull String viewerDocumentStorageId,//
-        @Deprecated//
-        @Schema(description = "Path where the Section Grid will be stored.")//
-        @NonNull String sectionGridStorageId) {
+        @NonNull String viewerDocumentStorageId) {

 }
@@ -15,8 +15,8 @@ dependencies {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
-    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.40.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")

    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -3,20 +3,23 @@ package com.knecon.fforesight.service.layoutparser.processor;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
-import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@@ -35,7 +38,6 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
-import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
@@ -50,98 +52,171 @@ import com.knecon.fforesight.service.layoutparser.processor.services.mapper.Docu
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;

+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

+@SuppressWarnings("PMD.CloseResource")
@Slf4j
@Service
@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class LayoutParsingPipeline {

-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final CvTableParsingAdapter cvTableParsingAdapter;
-    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final SectionsBuilderService sectionsBuilderService;
-    private final SectionGridCreatorService sectionGridCreatorService;
-    private final TaasClassificationService taasClassificationService;
-    private final RedactManagerClassificationService redactManagerClassificationService;
-    private final DocuMineClassificationService docuMineClassificationService;
-    private final SimplifiedSectionTextService simplifiedSectionTextService;
-    private final BodyTextFrameService bodyTextFrameService;
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final TaasBlockificationService taasBlockificationService;
-    private final DocuMineBlockificationService docuMineBlockificationService;
-    private final RedactManagerBlockificationService redactManagerBlockificationService;
-    private final ViewerDocumentService viewerDocumentService;
+    ImageServiceResponseAdapter imageServiceResponseAdapter;
+    CvTableParsingAdapter cvTableParsingAdapter;
+    LayoutParsingStorageService layoutParsingStorageService;
+    SectionsBuilderService sectionsBuilderService;
+    TaasClassificationService taasClassificationService;
+    RedactManagerClassificationService redactManagerClassificationService;
+    DocuMineClassificationService docuMineClassificationService;
+    SimplifiedSectionTextService simplifiedSectionTextService;
+    BodyTextFrameService bodyTextFrameService;
+    RulingCleaningService rulingCleaningService;
+    TableExtractionService tableExtractionService;
+    TaasBlockificationService taasBlockificationService;
+    DocuMineBlockificationService docuMineBlockificationService;
+    RedactManagerBlockificationService redactManagerBlockificationService;
+    ViewerDocumentService viewerDocumentService;
+    ObservationRegistry observationRegistry;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {

        long start = System.currentTimeMillis();
+        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

-        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
-            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
-            }
+        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
+        File viewerDocumentFile = File.createTempFile("viewer_document", ".pdf");

-            TableServiceResponse tableServiceResponse = new TableServiceResponse();
-            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
-            }
-
-            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
-            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-
-            int numberOfPages = originDocument.getNumberOfPages();
-
-            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
-            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
-
-            try (var out = new ByteArrayOutputStream()) {
-                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out, false);
-                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
-            }
-
-            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
-                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
-                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
-            }
-
-            return LayoutParsingFinishedEvent.builder()
-                    .identifier(layoutParsingRequest.identifier())
-                    .numberOfPages(numberOfPages)
-                    .duration(System.currentTimeMillis() - start)
-                    .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
-                            layoutParsingRequest.structureFileStorageId(),
-                            layoutParsingRequest.textBlockFileStorageId(),
-                            layoutParsingRequest.positionBlockFileStorageId(),
-                            layoutParsingRequest.pageFileStorageId()))
-                    .build();
+        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
        }
+
+        TableServiceResponse tableServiceResponse = new TableServiceResponse();
+        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+        }
+
+        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+                originFile,
+                imageServiceResponse,
+                tableServiceResponse,
+                layoutParsingRequest.identifier().toString());
+        log.info("Building document graph for {}", layoutParsingRequest.identifier());
+
+        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
+
+        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
+        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
+        viewerDocumentService.createViewerDocument(originFile, documentGraph, viewerDocumentFile, false);
+        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+            log.info("Building research document data for {}", layoutParsingRequest.identifier());
+            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+        }
+
+        originFile.delete();
+        viewerDocumentFile.delete();
+
+        return LayoutParsingFinishedEvent.builder()
+                .identifier(layoutParsingRequest.identifier())
+                .numberOfPages(documentGraph.getNumberOfPages())
+                .duration(System.currentTimeMillis() - start)
+                .message(format("""
+                                Layout parsing has finished in %.02f s.
+                                identifiers: %s
+                                %s
+                                Files have been saved with Ids:
+                                Structure: %s
+                                Text: %s
+                                Positions: %s
+                                PageData: %s
+                                Simplified Text: %s
+                                Viewer Doc: %s""",
+                        ((float) (System.currentTimeMillis() - start)) / 1000,
+                        layoutParsingRequest.identifier(),
+                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
+                        layoutParsingRequest.structureFileStorageId(),
+                        layoutParsingRequest.textBlockFileStorageId(),
+                        layoutParsingRequest.positionBlockFileStorageId(),
+                        layoutParsingRequest.pageFileStorageId(),
+                        layoutParsingRequest.simplifiedTextStorageId(),
+                        layoutParsingRequest.viewerDocumentStorageId()))
+                .build();
+
+    }
+
+
+    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
+
+        AtomicReference<Document> documentReference = new AtomicReference<>();
+
+        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
+            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
+        });
+
+        return documentReference.get();
+    }
+
+
+    private String buildSemanticNodeCountMessage(int numberOfPages, Map<NodeType, Long> semanticNodeCounts) {
+
+        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
+                numberOfPages,
+                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
    }


    @SneakyThrows
+    @Observed(name = "LayoutParsingPipeline", contextualName = "parse-layout")
    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
-                                              PDDocument originDocument,
+                                              File originFile,
                                              ImageServiceResponse imageServiceResponse,
-                                              TableServiceResponse tableServiceResponse) {
+                                              TableServiceResponse tableServiceResponse,
+                                              String identifier) {

+        PDDocument originDocument = openDocument(originFile);
+        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);

        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

-        originDocument.setAllSecurityToBeRemoved(true);
        long pageCount = originDocument.getNumberOfPages();

        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+
+            if (pageNumber % 100 == 0) {
+                // re-open document every once in a while to save on RAM. This has no significant performance impact.
+                // This is due to PDFBox caching all images and some other stuff with Soft References. This dereferences them and forces the freeing of memory.
+                originDocument.close();
+                originDocument = openDocument(originFile);
+            }
+
+            if (pageNumber % 100 == 0 || pageNumber == pageCount || pageNumber == 1) {
+                log.info("Extracting text on Page {} for {}", pageNumber, identifier);
+            }
+
            classificationDocument.setPages(classificationPages);
            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
            PDPage pdPage = originDocument.getPage(pageNumber - 1);
@@ -160,18 +235,12 @@ public class LayoutParsingPipeline {
            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);

            PDRectangle cropbox = pdPage.getCropBox();
-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                    stripper.getRulings(),
-                    stripper.getMinCharWidth(),
-                    stripper.getMaxCharHeight());
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());

            ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER ->
-                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case TAAS ->
-                        taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case DOCUMINE ->
-                        docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
            };
            classificationPage.setCleanRulings(cleanRulings);
            classificationPage.setRotation(rotation);
@@ -189,35 +258,59 @@ public class LayoutParsingPipeline {
                imageServiceResponseAdapter.findOcr(classificationPage);
            }

-            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            tableExtractionService.extractTables(cleanRulings, classificationPage);
+
            buildPageStatistics(classificationPage);
            increaseDocumentStatistics(classificationPage, classificationDocument);

            classificationPages.add(classificationPage);
        }

-        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+        originDocument.close();

+        log.info("Calculating BodyTextFrame for {}", identifier);
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+        log.info("Classify TextBlocks for {}", identifier);
        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }

+        log.info("Building Sections for {}", identifier);
        sectionsBuilderService.buildSections(classificationDocument);
        sectionsBuilderService.addImagesToSections(classificationDocument);
-
        return classificationDocument;
    }


-    private Map<String, Rectangle2D> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
-        Map<String, Rectangle2D> markedContentBboxes = new HashMap<>();
+    private void addNumberOfPagesToTrace(int numberOfPages, long size) {
+
+        if (observationRegistry.getCurrentObservation() != null) {
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileSize", String.valueOf(size));
+        }
+    }
+
+
+    @SneakyThrows
+    private PDDocument openDocument(File originFile) {
+
+        PDDocument document = Loader.loadPDF(originFile);
+        document.setAllSecurityToBeRemoved(true);
+        return document;
+    }
+
+
+    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+
+        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
        return markedContentBboxes;
    }

+
    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {

        if (!classificationPage.isLandscape()) {
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor;

-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileOutputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -11,13 +9,11 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;

-import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
@@ -27,6 +23,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -40,16 +37,18 @@ public class LayoutParsingStorageService {
    private final ObjectMapper objectMapper;


-    public PDDocument getOriginFile(String storageId) throws IOException {
+    public PDDocument getOriginDocument(String storageId) throws IOException {

-        try (var originDocumentInputStream = getObject(storageId)) {
-            File tempFile = createTempFile("document", ".pdf");
-            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
-                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
-                originDocumentInputStream.close();
-            }
-            return Loader.loadPDF(tempFile);
-        }
+        return Loader.loadPDF(getOriginFile(storageId));
+    }
+
+
+    @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
+    public File getOriginFile(String storageId) throws IOException {
+
+        File tempFile = createTempFile("document", ".pdf");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return tempFile;
    }


@@ -75,6 +74,7 @@ public class LayoutParsingStorageService {
    }


+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
@@ -84,12 +84,6 @@ public class LayoutParsingStorageService {
    }


-    public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) {
-
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid);
-    }
-
-
    public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.researchDocumentStorageId(), researchDocumentData);
@@ -121,6 +115,7 @@ public class LayoutParsingStorageService {
    }


+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-simplified-text")
    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
@@ -138,9 +133,10 @@ public class LayoutParsingStorageService {


    @SneakyThrows
-    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-viewer-document")
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, File out) {

-        try (var in = new ByteArrayInputStream(out.toByteArray())) {
+        try (var in = new FileInputStream(out)) {

            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
        }
@@ -72,9 +72,27 @@ public abstract class AbstractPageBlock {
    }


-    public boolean intersectsY(AbstractPageBlock atc) {
+    public boolean intersectsY(AbstractPageBlock apb) {

-        return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY();
+        return this.minY <= apb.getMaxY() && this.maxY >= apb.getMinY();
+    }
+
+
+    public boolean almostIntersects(AbstractPageBlock apb, float yThreshold, float xThreshold) {
+
+        return this.almostIntersectsX(apb, xThreshold) && this.almostIntersectsY(apb, yThreshold);
+    }
+
+
+    private boolean almostIntersectsY(AbstractPageBlock apb, float threshold) {
+
+        return this.minY - threshold <= apb.getMaxY() && this.maxY + threshold >= apb.getMinY();
+    }
+
+
+    private boolean almostIntersectsX(AbstractPageBlock apb, float threshold) {
+
+        return this.minX - threshold <= apb.getMaxX() && this.maxX + threshold >= apb.getMinX();
    }


@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.ArrayList;
 import java.util.List;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText;

@@ -25,7 +24,6 @@ public class ClassificationDocument {
    private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
    private boolean headlines;

-    private SectionGrid sectionGrid = new SectionGrid();
    private long rulesVersion;

 }
@@ -42,6 +42,6 @@ public class ClassificationPage {

    CleanRulings cleanRulings;

-    private Map<String, Rectangle2D> markedContentBboxPerType = new HashMap<>();
+    private Map<String, List<Rectangle2D>> markedContentBboxPerType = new HashMap<>();

 }
@@ -10,7 +10,6 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

-import com.amazonaws.services.kms.model.NotFoundException;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -84,7 +83,7 @@ public class Document implements GenericSemanticNode {
    @Override
    public Headline getHeadline() {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build());
    }


@@ -100,6 +99,12 @@ public class Document implements GenericSemanticNode {
    }


+    public Map<NodeType, Long> buildSemanticNodeCounts() {
+
+        return streamAllSubNodes().collect(Collectors.groupingBy(SemanticNode::getType, Collectors.counting()));
+    }
+
+
    @Override
    public String toString() {

@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Footer implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -62,4 +67,14 @@ public class Footer implements GenericSemanticNode {
        return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Header implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public boolean isLeaf() {
@@ -62,4 +67,14 @@ public class Header implements GenericSemanticNode {
        return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Headline implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -69,4 +74,14 @@ public class Headline implements GenericSemanticNode {
        return this;
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -32,6 +34,9 @@ public class Paragraph implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -60,4 +65,14 @@ public class Paragraph implements GenericSemanticNode {
        return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -35,6 +37,9 @@ public class Section implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -74,4 +79,14 @@ public class Section implements GenericSemanticNode {
                .orElseGet(() -> getParent().getHeadline());
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -398,12 +398,11 @@ public interface SemanticNode {
     */
    default Map<Page, Rectangle2D> getBBox() {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        if (isLeaf()) {
-            return getBBoxFromLeafTextBlock(bBoxPerPage);
+            return getBBoxFromLeafTextBlock();
        }

-        return getBBoxFromChildren(bBoxPerPage);
+        return getBBoxFromChildren();
    }


@@ -426,25 +425,31 @@ public interface SemanticNode {

    /**
     * TODO: this produces unwanted results for sections spanning multiple columns.
-     *
-     * @param bBoxPerPage initial empty BoundingBox
+     * Computes the Union of the bounding boxes of all children recursively.
     * @return The union of the BoundingBoxes of all children
     */
-    private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {
+    private Map<Page, Rectangle2D> getBBoxFromChildren() {

-        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
-            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
-            return map2;
-        }).orElse(bBoxPerPage);
+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
+        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
+        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
+        for (Page page : pages) {
+            Rectangle2D bBoxOnPage = childrenBBoxes.stream()
+                    .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
+                    .map(childBboxPerPage -> childBboxPerPage.get(page))
+                    .collect(RectangleTransformations.collectBBox());
+            bBoxPerPage.put(page, bBoxOnPage);
+        }
+        return bBoxPerPage;
    }


    /**
-     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of all BoundingBoxes of the TextBlock of this node
     */
-    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
+    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {

+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
        return bBoxPerPage;
@@ -2,10 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

 import static java.lang.String.format;

+import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -34,13 +36,14 @@ public class Table implements SemanticNode {

    int numberOfRows;
    int numberOfCols;
-
    TextBlock textBlock;

    @Builder.Default
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;

    /**
     * Streams all entities in this table, that appear in a row, which contains any of the provided strings.
@@ -208,7 +211,6 @@ public class Table implements SemanticNode {
        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
    }

-
    /**
     * Streams all TableCells row-wise and filters them with header == true.
     *
@@ -313,5 +315,12 @@ public class Table implements SemanticNode {

        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
    }
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {

+        if (bBoxCache == null) {
+            bBoxCache = SemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
 }
@@ -1,12 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -252,7 +254,8 @@ public class TablePageBlock extends AbstractPageBlock {
                if (prevY != null && prevX != null) {
                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));

-                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
+                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
+
                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
                    if (cell.hasMinimumSize()) {
                        row.add(cell);
@@ -273,6 +276,21 @@ public class TablePageBlock extends AbstractPageBlock {
    }


+
+    public boolean intersects(Cell cell1, Cell cell2) {
+        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
+            return false;
+        }
+        double x0 = cell1.getX() + 2;
+        double y0 = cell1.getY() + 2;
+        return (cell2.x + cell2.width > x0 &&
+                cell2.y + cell2.height > y0 &&
+                cell2.x < x0 + cell1.getWidth() -2 &&
+                cell2.y < y0 + cell1.getHeight() -2);
+    }
+
+
+
    @Override
    public String getText() {

@@ -17,7 +17,6 @@ import lombok.SneakyThrows;
@AllArgsConstructor
 public class RedTextPosition {

-    private String textMatrix;
    private float[] position;

    @JsonIgnore
@@ -56,8 +55,6 @@ public class RedTextPosition {

        pos.setFontSizeInPt(textPosition.getFontSizeInPt());

-        pos.setTextMatrix(textPosition.getTextMatrix().toString());
-
        var position = new float[4];

        position[0] = textPosition.getXDirAdj();
@@ -9,8 +9,6 @@ import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;

-import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;

@@ -142,8 +140,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minX value
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMinXDirAdj() {

        return textPositions.get(0).getXDirAdj();
@@ -157,8 +154,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxX value
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMaxXDirAdj() {

        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
@@ -172,8 +168,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMinYDirAdj() {

        return textPositions.get(0).getYDirAdj() - getTextHeight();
@@ -187,8 +182,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMaxYDirAdj() {

        return textPositions.get(0).getYDirAdj();
@@ -196,42 +190,38 @@ public class TextPositionSequence implements CharSequence {
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getHeight() {

        return getMaxYDirAdj() - getMinYDirAdj();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getWidth() {

        return getMaxXDirAdj() - getMinXDirAdj();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public String getFont() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "none";
+        }
        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public String getFontStyle() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "standard";
+        }
        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);

        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
@@ -243,20 +233,15 @@ public class TextPositionSequence implements CharSequence {
        } else {
            return "standard";
        }
-
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getFontSize() {

        return textPositions.get(0).getFontSizeInPt();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getSpaceWidth() {

        return textPositions.get(0).getWidthOfSpace();
@@ -272,8 +257,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return bounding box of the word in Pdf Coordinate System
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    @SneakyThrows
    public Rectangle getRectangle() {

@@ -8,6 +8,7 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.PageInfo;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;

@@ -24,24 +25,26 @@ public class CvTableParsingAdapter {
        Map<Integer, List<TableCells>> tableCells = new HashMap<>();
        tableServiceResponse.getData()
                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells())));
+                        .addAll(convertTableCells(tableData.getTableCells(), tableData.getPageInfo())));

        return tableCells;
    }


-    private Collection<TableCells> convertTableCells(List<TableCells> tableCells) {
+    private Collection<TableCells> convertTableCells(List<TableCells> tableCells, PageInfo pageInfo) {

        List<TableCells> cvParsedTableCells = new ArrayList<>();

-        tableCells.forEach(t -> cvParsedTableCells.add(TableCells.builder()
-                .y0(t.getY0())
-                .x1(t.getX1())
-                .y1(t.getY1())
-                .x0(t.getX0())
-                .width(t.getWidth())
-                .height(t.getHeight())
-                .build()));
+        tableCells.stream()
+                .filter(cell -> cell.getWidth() < pageInfo.getWidth() * 0.98 && cell.getHeight() < pageInfo.getHeight() * 0.98)
+                .forEach(t -> cvParsedTableCells.add(TableCells.builder()
+                        .y0(t.getY0())
+                        .x1(t.getX1())
+                        .y1(t.getY1())
+                        .x0(t.getX0())
+                        .width(t.getWidth())
+                        .height(t.getHeight())
+                        .build()));

        return cvParsedTableCells;
    }
@@ -12,9 +12,9 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;

 import lombok.RequiredArgsConstructor;
@@ -25,10 +25,13 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RulingCleaningService {

-    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
+    private static final float THRESHOLD = 6;
+
+
+    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {

        if (!rulings.isEmpty()) {
-            snapPoints(rulings, minCharWidth, maxCharHeight);
+            snapPoints(rulings);
        }

        List<Ruling> vrs = new ArrayList<>();
@@ -53,14 +56,11 @@ public class RulingCleaningService {
        }
        List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);

-        return CleanRulings.builder()
-                .vertical(verticalRulingLines)
-                .horizontal(horizontalRulingLines)
-                .build();
+        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
    }


-    public void snapPoints(List<? extends Line2D.Float> rulings, float xThreshold, float yThreshold) {
+    public void snapPoints(List<? extends Line2D.Float> rulings) {

        // collect points and keep a Line -> p1,p2 map
        Map<Line2D.Float, Point2D[]> linesToPoints = new HashMap<>();
@@ -81,7 +81,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < xThreshold) {
+            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -108,7 +108,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < yThreshold) {
+            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -1,146 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BinaryOperator;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collector;
-import java.util.stream.Stream;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class SectionGridCreatorService {
-
-    public SectionGrid createSectionGrid(Document document) {
-
-        Map<Integer, List<SectionRectangle>> sectionBBox = document.streamAllSubNodesOfType(NodeType.SECTION).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> paragraphBBox = document.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> headlineBBox = document.streamAllSubNodesOfType(NodeType.HEADLINE).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> tableBBox = document.streamAllSubNodesOfType(NodeType.TABLE).map(node -> (Table) node).collect(new TableGridCollector());
-        var sectionGrid = new SectionGrid();
-
-        sectionGrid.setRectanglesPerPage(mergeMapsByConcatenatingLists(//
-                mergeMapsByConcatenatingLists(paragraphBBox, headlineBBox), //
-                mergeMapsByConcatenatingLists(sectionBBox, tableBBox)));
-
-        return sectionGrid;
-    }
-
-
-    private static abstract class GridCollector<T> implements Collector<T, Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> {
-
-        @Override
-        public Supplier<Map<Integer, List<SectionRectangle>>> supplier() {
-
-            return HashMap::new;
-        }
-
-
-        @Override
-        public Function<Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> finisher() {
-
-            return Function.identity();
-        }
-
-
-        @Override
-        public BinaryOperator<Map<Integer, List<SectionRectangle>>> combiner() {
-
-            return SectionGridCreatorService::mergeMapsByConcatenatingLists;
-        }
-
-
-        @Override
-        public Set<Characteristics> characteristics() {
-
-            return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
-        }
-
-    }
-
-    private static class TableGridCollector extends GridCollector<Table> {
-
-        @Override
-        public BiConsumer<Map<Integer, List<SectionRectangle>>, Table> accumulator() {
-
-            return (map, table) -> table.getPages()
-                    .forEach(page -> map.merge(page.getNumber(), List.of(toSectionRectangle(table, page, table.getPages().size())), SectionGridCreatorService::concatLists));
-        }
-
-
-        private static SectionRectangle toSectionRectangle(Table table, Page page, int numberOfParts) {
-
-            Rectangle2D rect = table.getBBox().get(page);
-            List<CellRectangle> tableCellRectangles = table.streamTableCells()
-                    .map(TableCell::getBBox)
-                    .map(map -> map.get(page))
-                    .filter(Objects::nonNull)
-                    .map(rectangle2D -> new CellRectangle(new Point((float) rectangle2D.getX(), (float) rectangle2D.getY()),
-                            (float) rectangle2D.getWidth(),
-                            (float) rectangle2D.getHeight()))
-                    .toList();
-            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()),
-                    (float) rect.getWidth(),
-                    (float) rect.getHeight(),
-                    1,
-                    numberOfParts,
-                    tableCellRectangles);
-        }
-
-    }
-
-    private static class SectionGridCollector extends GridCollector<Map<Page, Rectangle2D>> {
-
-        @Override
-        public BiConsumer<Map<Integer, List<SectionRectangle>>, Map<Page, Rectangle2D>> accumulator() {
-
-            return (mapToKeep, mapToMerge) -> mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page.getNumber(),
-                    List.of(toSectionRectangle(rectangle, mapToMerge.values().size())),
-                    SectionGridCreatorService::concatLists));
-
-        }
-
-
-        private static SectionRectangle toSectionRectangle(Rectangle2D rect, int numberOfParts) {
-
-            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()), (float) rect.getWidth(), (float) rect.getHeight(), 1, numberOfParts, null);
-        }
-
-    }
-
-
-    private static Map<Integer, List<SectionRectangle>> mergeMapsByConcatenatingLists(Map<Integer, List<SectionRectangle>> mapToKeep,
-                                                                                      Map<Integer, List<SectionRectangle>> mapToMerge) {
-
-        mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page, rectangle, SectionGridCreatorService::concatLists));
-        return mapToKeep;
-    }
-
-
-    private static List<SectionRectangle> concatLists(List<SectionRectangle> l1, List<SectionRectangle> l2) {
-
-        return Stream.concat(l1.stream(), l2.stream()).toList();
-    }
-
-}
@@ -12,7 +12,6 @@ import java.util.Set;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -66,6 +65,17 @@ public class TableExtractionService {
    };


+    public boolean contains(Cell cell, double x, double y, double w, double h) {
+
+        if (cell.isEmpty() || w <= 0 || h <= 0) {
+            return false;
+        }
+        double x0 = cell.getX();
+        double y0 = cell.getY();
+        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
+    }
+
+
    /**
     * Finds tables on a page and moves textblocks into cells of the found tables.
     * Note: This algorithm uses Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -79,16 +89,17 @@ public class TableExtractionService {
     * @param cleanRulings The lines used to build the table.
     * @param page         Page object that contains textblocks and statistics.
     */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page, LayoutParsingType layoutParsingType) {
+    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {

-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical(), layoutParsingType);
+        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());

        List<TextPageBlock> toBeRemoved = new ArrayList<>();

        for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
            TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
            for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
+                if (cell.hasMinimumSize() && contains(cell,
+                        textBlock.getPdfMinX(),
                        textBlock.getPdfMinY(),
                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -102,7 +113,7 @@ public class TableExtractionService {
        cells = new ArrayList<>(new HashSet<>(cells));
        DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);

-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).toList();
+        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);

        List<TablePageBlock> tables = new ArrayList<>();
        for (Rectangle area : spreadsheetAreas) {
@@ -135,16 +146,14 @@ public class TableExtractionService {
    }


-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, LayoutParsingType layoutParsingType) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

-        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
-            // TODO: breaks some tables, for example "1 Abamectin Prr.pdf" try to fix this upstream in RulingCleaningService
-            for (Ruling r : horizontalRulingLines) {
-                if (r.getX2() < r.getX1()) {
-                    double a = r.getX2();
-                    r.x2 = (float) r.getX1();
-                    r.x1 = (float) a;
-                }
+        // Fix for 211.pdf
+        for (Ruling r : horizontalRulingLines) {
+            if (r.getX2() < r.getX1()) {
+                double a = r.getX2();
+                r.x2 = (float) r.getX1();
+                r.x1 = (float) a;
            }
        }

@@ -1,13 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;

-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;

-import org.springframework.stereotype.Service;
+// TODO: figure out, why this fails the build
+// import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -16,14 +11,25 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;

@Service
@SuppressWarnings("all")
 public class TaasBlockificationService {

    private static final float THRESHOLD = 1f;
-    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f;
+    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f; // multiplied with text height
+    private static final float INTERSECTS_Y_THRESHOLD = 4;// 2 * HEIGHT_PADDING // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
    private static final int X_GAP_SPLIT_CONSTANT = 50;
+    public static final int X_ALIGNMENT_THRESHOLD = 1;
+    public static final int NEGATIVE_X_GAP_THRESHOLD = -5;
+
+    private Pattern listIdentifier = Pattern.compile("^(?:(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)]))|\\uF0B7", Pattern.CASE_INSENSITIVE);


    /**
@@ -39,14 +45,28 @@ public class TaasBlockificationService {
    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

        List<TextPageBlock> classificationTextBlocks = constructFineGranularTextPageBlocks(textPositions, horizontalRulingLines, verticalRulingLines);
-
-        classificationTextBlocks = mergeFineGranularTextPageBlocks(classificationTextBlocks);
+        classificationTextBlocks = mergeTextPageBlocksAligningX(classificationTextBlocks);
+        classificationTextBlocks = mergeIntersectingTextBlocksUntilConvergence(classificationTextBlocks);

        return new ClassificationPage(new ArrayList<>(classificationTextBlocks.stream().map(classificationTextBlock -> (AbstractPageBlock) classificationTextBlock).toList()));
    }


-    private List<TextPageBlock> mergeFineGranularTextPageBlocks(List<TextPageBlock> classificationTextBlocks) {
+    private List<TextPageBlock> mergeIntersectingTextBlocksUntilConvergence(List<TextPageBlock> classificationTextBlocks) {
+
+        int currentSize = classificationTextBlocks.size();
+        while (true) {
+            classificationTextBlocks = mergeTextPageBlocksAlmostIntersecting(classificationTextBlocks);
+            if (classificationTextBlocks.size() == currentSize) {
+                break;
+            }
+            currentSize = classificationTextBlocks.size();
+        }
+        return classificationTextBlocks;
+    }
+
+
+    private List<TextPageBlock> mergeTextPageBlocksAligningX(List<TextPageBlock> classificationTextBlocks) {

        if (classificationTextBlocks.isEmpty()) {
            return new ArrayList<>();
@@ -55,16 +75,29 @@ public class TaasBlockificationService {
        List<TextPageBlock> currentTextBlocksToMerge = new LinkedList<>();
        textBlocksToMerge.add(currentTextBlocksToMerge);
        TextPageBlock previousTextBlock = null;
+        Float lastLineGap = null;
        for (TextPageBlock currentTextBlock : classificationTextBlocks) {
            if (previousTextBlock == null) {
                currentTextBlocksToMerge.add(currentTextBlock);
                previousTextBlock = currentTextBlock;
                continue;
            }
-            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < 1;
-            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < 5;
-            if (alignsXRight && smallYGap) {
+
+
+            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
+            boolean isListIdentifier = listIdentifierPattern.find();
+
+            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+
+            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
+//            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+
+            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
+            boolean alignsXLeft = Math.abs(currentTextBlock.getPdfMinX() - previousTextBlock.getPdfMinX()) < X_ALIGNMENT_THRESHOLD;
+//            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < yGap;
+            if (yGap && sameFont && !isListIdentifier) {
                currentTextBlocksToMerge.add(currentTextBlock);
+
            } else {
                currentTextBlocksToMerge = new LinkedList<>();
                currentTextBlocksToMerge.add(currentTextBlock);
@@ -76,6 +109,23 @@ public class TaasBlockificationService {
    }


+    private List<TextPageBlock> mergeTextPageBlocksAlmostIntersecting(List<TextPageBlock> textPageBlocks) {
+
+        Set<TextPageBlock> alreadyMerged = new HashSet<>();
+        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
+        for (TextPageBlock textPageBlock : textPageBlocks) {
+            if (alreadyMerged.contains(textPageBlock)) {
+                continue;
+            }
+            alreadyMerged.add(textPageBlock);
+            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
+                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
+                    .toList());
+        }
+        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
+    }
+
+
    private void assignOrientations(List<TextPageBlock> classificationTextBlocks) {

        Iterator<TextPageBlock> itty = classificationTextBlocks.iterator();
@@ -128,8 +178,8 @@ public class TaasBlockificationService {


    private List<TextPageBlock> constructFineGranularTextPageBlocks(List<TextPositionSequence> textPositions,
-                                                                                        List<Ruling> horizontalRulingLines,
-                                                                                        List<Ruling> verticalRulingLines) {
+                                                                    List<Ruling> horizontalRulingLines,
+                                                                    List<Ruling> verticalRulingLines) {

        int indexOnPage = 0;
        List<TextPositionSequence> wordClusterToCombine = new ArrayList<>();
@@ -138,18 +188,18 @@ public class TaasBlockificationService {
        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
        TextPositionSequence prev = null;
        // TODO: make static final constant
-        var listIdentitifier = Pattern.compile("\\b(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)])", Pattern.CASE_INSENSITIVE);
+

        boolean wasSplitted = false;
        Float splitX1 = null;
        for (TextPositionSequence word : textPositions) {

-            Matcher listIdentifierPattern = listIdentitifier.matcher(word.toString());
+            Matcher listIdentifierPattern = listIdentifier.matcher(word.toString());

            boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
            boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());
            boolean positiveXGapInline = prev != null && maxX + X_GAP_SPLIT_CONSTANT < word.getMinXDirAdj() && sameLine;
-            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < -5;
+            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < NEGATIVE_X_GAP_THRESHOLD;
            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
            boolean splitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
@@ -164,7 +214,7 @@ public class TaasBlockificationService {

                Orientation prevOrientation = null;
                if (!classificationTextBlocks.isEmpty()) {
-                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - 1).getOrientation();
+                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - X_ALIGNMENT_THRESHOLD).getOrientation();
                }

                TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);
@@ -133,7 +133,7 @@ public class SearchTextWithTextPositionFactory {

    private static void addTextPositionWithFontType(RedTextPosition currentTextPosition, String fontType, List<Integer> fontTypePositions, int stringIdx) {

-        if (currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
+        if (currentTextPosition.getFontName() != null && currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
            fontTypePositions.add(stringIdx);
        }
    }
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -8,6 +9,7 @@ import java.util.Map;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;

@@ -1,18 +1,34 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.parsing;

-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.color.CMMException;
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.*;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
 import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
 import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
-import org.apache.pdfbox.contentstream.operator.state.*;
+import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
+import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
+import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
+import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
+import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
+import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
+import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSNumber;
@@ -21,11 +37,14 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.text.TextPosition;

-import java.awt.color.CMMException;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;

@Getter
@Slf4j
@@ -36,11 +55,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private final List<Ruling> graphicsPath = new ArrayList<>();
    @Setter
    protected PDPage pdpage;
-    private int minCharWidth;
-    private int maxCharWidth;
-    private int minCharHeight;
-    private int maxCharHeight;
-

    private float path_x;
    private float path_y;
@@ -73,7 +87,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        this.addOperator(new SetFontAndSize(this));
        this.addOperator(new SetLineWidth(this));

-
        addOperator(new BeginMarkedContentSequenceWithProperties(this));
 //        addOperator(new BeginMarkedContentSequence(this));
        addOperator(new EndMarkedContentSequence(this));
@@ -232,33 +245,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
            }

-            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
-            if (charWidth < minCharWidth) {
-                minCharWidth = charWidth;
-            }
-            if (charWidth > maxCharWidth) {
-                maxCharWidth = charWidth;
-            }
-
-            int charHeight = (int) textPositions.get(i).getHeightDir();
-            if (charHeight < minCharHeight) {
-                minCharHeight = charHeight;
-            }
-            if (charWidth > maxCharHeight) {
-                maxCharHeight = charHeight;
-            }
-
            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
                startIndex++;
                continue;
            }

            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj()) {
+            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -266,9 +261,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -278,13 +271,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                    .getUnicode()
                    .equals("\t")) && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {

                    // Remove false sequence ends (whitespaces)
-                    if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                            .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
+                    if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
                        for (TextPosition t : sublist) {
                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                        }
@@ -319,13 +309,29 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    }


+    public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {
+
+        return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
+    }
+
+
+    public boolean checkIfSequenceContainsOnlyWhitespaces(List<TextPosition> sublist) {
+
+        return !(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                .getUnicode()
+                .equals("\t")));
+    }
+
+
+    public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {
+
+        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
+                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
+    }
+
    @Override
    public String getText(PDDocument doc) throws IOException {

-        minCharWidth = Integer.MAX_VALUE;
-        maxCharWidth = 0;
-        minCharHeight = Integer.MAX_VALUE;
-        maxCharHeight = 0;
        textPositionSequences.clear();
        rulings.clear();
        graphicsPath.clear();
@@ -1711,7 +1711,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
        int numberOfStrings = line.size();
        for (int i = 0; i < numberOfStrings; i++) {
            WordWithTextPositions word = line.get(i);
-            word.getTextPositions().sort(Comparator.comparing(TextPosition::getX));
+            word.getTextPositions().sort(Comparator.comparing(TextPosition::getXDirAdj));
            writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
            if (i < numberOfStrings - 1) {
                writeWordSeparator();
@@ -2,12 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor.services.visualizat

 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
+import java.io.File;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.util.HashSet;
-import java.util.Set;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;

-import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
@@ -31,6 +32,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.visualization.
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;

+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -40,30 +44,31 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class ViewerDocumentService {

-
    private static final String LAYER_NAME = "Layout grid";
    private static final int FONT_SIZE = 10;
    public static final float LINE_WIDTH = 1f;

    private final LayoutGridService layoutGridService;
+    private final ObservationRegistry observationRegistry;


    @SneakyThrows
-    public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream, boolean layerVisibilityDefaultValue) {
+    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
+    public void createViewerDocument(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {

-        log.info("Start Viewer Document Creation");
+        Path tmpFile = Files.createTempFile("tmpViewerDocument", ".pdf");
+        PDDocument pdDocument = openPDDocument(originFile);
        LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
-        // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
-        // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
-        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
-        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
+
+        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
        PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);

        for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
-            PDPage pdPage = pdDocument.getPage(pageNumber);

+            PDPage pdPage = pdDocument.getPage(pageNumber);
+//
            AffineTransform textDeRotationMatrix = getTextDeRotationTransform(pdPage);
-            addLayerToPageRessources(pdPage);
+            addLayerToPageResources(pdPage);

            // We need to save the graphics state before, such that our appended content cannot be affected by previous content streams with side effects,
            // e.g. not escaped matrix transformations.
@@ -116,17 +121,48 @@ public class ViewerDocumentService {
                contentStream.restoreGraphicsState();
                contentStream.endMarkedContent();
            }
-            dictionariesToUpdate.add(pdPage.getCOSObject());
-            dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
+
+            if (pageNumber % 500 == 0 && pageNumber != 0) { // re-open document every once in a while to save on RAM
+                log.info("Incremental save after {} pages", pageNumber);
+                observedIncrementalSave(pdDocument, destinationFile);
+                pdDocument.close();
+                Files.copy(destinationFile.toPath(), tmpFile, StandardCopyOption.REPLACE_EXISTING);
+                pdDocument = openPDDocument(tmpFile.toFile());
+                layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
+            }
+
        }
-        dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
-//        dictionariesToUpdate.add(pdDocument.getDocument().getTrailer());
-        pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
-        log.info("Saved Viewer Document");
+        observedIncrementalSave(pdDocument, destinationFile);
+
+        tmpFile.toFile().delete();
+        pdDocument.close();
    }


-    private static void addLayerToPageRessources(PDPage pdPage) {
+    private static PDDocument openPDDocument(File tmpFile) throws IOException {
+
+        PDDocument pdDocument;
+        pdDocument = Loader.loadPDF(tmpFile);
+        pdDocument.setAllSecurityToBeRemoved(true);
+        return pdDocument;
+    }
+
+
+    @SneakyThrows
+    private void observedIncrementalSave(PDDocument pdDocument, File outputFile) {
+
+        Observation.createNotStarted("ViewerDocumentService", observationRegistry).contextualName("incremental-save").observe(() -> {
+            try {
+                pdDocument.save(outputFile);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+    }
+
+
+    private static void addLayerToPageResources(PDPage pdPage) {

        PDResources resources = pdPage.getResources();
        if (resources == null) {
@@ -147,7 +183,7 @@ public class ViewerDocumentService {
    }


-    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate, boolean layerVisibilityDefaultValue) {
+    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, boolean layerVisibilityDefaultValue) {

        PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
        PDOptionalContentProperties ocprops = catalog.getOCProperties();
@@ -163,7 +199,7 @@ public class ViewerDocumentService {
            ocprops.addGroup(layer);
        }
        ocprops.setGroupEnabled(layer, layerVisibilityDefaultValue);
-        dictionariesToUpdate.add(catalog.getCOSObject());
+//        dictionariesToUpdate.add(catalog.getCOSObject());
        return layer;
    }

@@ -9,6 +9,7 @@ import org.apache.pdfbox.text.TextPosition;

 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -19,10 +20,10 @@ public class MarkedContentUtils {
    public static final String HEADER = "Header";
    public static final String FOOTER = "Footer";

-    public Rectangle2D getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {
+    public List<Rectangle2D> getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {

        if (markedContents == null) {
-            return null;
+            return Collections.emptyList();
        }

        var markedContentByYPosition = markedContents.stream()
@@ -37,19 +38,19 @@ public class MarkedContentUtils {
                .collect(Collectors.groupingBy(TextPosition::getY));

        if (markedContentByYPosition.isEmpty()) {
-            return null;
+            return Collections.emptyList();
        }

-        return RectangleTransformations.rectangle2DBBox(markedContentByYPosition.values().stream()
+        return markedContentByYPosition.values().stream()
                .map(textPositions -> new TextPositionSequence(textPositions.stream()
                        .toList(), 0, true)
                        .getRectangle())
-                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList()));
+                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList());
    }


-    public boolean intersects(TextPageBlock textBlock, Map<String, Rectangle2D> markedContentBboxPerType, String type) {
-        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight());
+    public boolean intersects(TextPageBlock textBlock, Map<String, List<Rectangle2D>> markedContentBboxPerType, String type) {
+        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).stream().anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
    }

 }
@@ -19,9 +19,10 @@ public final class PositionUtils {

        double threshold = textBlock.getMostPopularWordHeight() * 3;

-        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX() && textBlock.getPdfMaxX() - threshold < btf.getTopLeft()
-                .getX() + btf.getWidth() && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY() && textBlock.getPdfMaxY() - threshold < btf.getTopLeft()
-                .getY() + btf.getHeight()) {
+        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX()
+                && textBlock.getPdfMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth()
+                && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY()
+                && textBlock.getPdfMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
            return true;
        } else {
            return false;
@@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
    application
    id("com.knecon.fforesight.java-conventions")
-    id("org.springframework.boot") version "3.1.3"
+    id("org.springframework.boot") version "3.1.5"
    id("io.spring.dependency-management") version "1.1.3"
    id("org.sonarqube") version "4.3.0.3225"
    id("io.freefair.lombok") version "8.2.2"
@@ -12,31 +12,43 @@ plugins {

 description = "layoutparser-service-server"

+configurations {
+    all {
+        exclude(group = "commons-logging", module = "commons-logging")
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
+        exclude(group = "com.iqser.red.commons", module = "logging-commons")
+    }
+}
+
+
 val jacksonVersion = "2.15.2"
 val pdfBoxVersion = "3.0.0"
+val springBootStarterVersion = "3.1.5"

 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.commons:storage-commons:2.40.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
+    implementation("com.knecon.fforesight:tracing-commons:0.3.0")
+    implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
    implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
+    implementation("net.logstash.logback:logstash-logback-encoder:7.4")

    // for integration testing only
    testImplementation(project(":layoutparser-service-internal-api"))

-    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
+    testImplementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0")
    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
-    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.3")
 }

 /*
@@ -5,6 +5,7 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
@@ -13,8 +14,11 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingService
 import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.aop.ObservedAspect;
+
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
-@Import({MetricsConfiguration.class, StorageAutoConfiguration.class,  LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
+@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

@@ -23,4 +27,11 @@ public class Application {
        SpringApplication.run(Application.class, args);
    }

+
+    @Bean
+    public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
+
+        return new ObservedAspect(observationRegistry);
+    }
+
 }
@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.queue;

+import java.util.Arrays;
+
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
@@ -45,12 +47,12 @@ public class MessageHandler {
        }
        LayoutParsingFinishedEvent layoutParsingFinishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent);
-        log.info("Layout parsing finished {} in {} ms", layoutParsingRequest.identifier(), layoutParsingFinishedEvent.duration());
    }


    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {

+        Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
    }

@@ -24,8 +24,7 @@ public class MessagingConfiguration {
    @Bean
    public Queue layoutparsingResponseQueue() {

-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
    }


@@ -5,10 +5,19 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/

 fforesight.tenants.remote: true

+logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
+
+logging.type: ${LOGGING_TYPE:CONSOLE}
+kubernetes.namespace: ${NAMESPACE:default}
+project.version: 1.0-SNAPSHOT
+
+
 server:
  port: 8080

 spring:
+  application:
+    name: layoutparser-service
  main:
    allow-circular-references: true # FIXME
  rabbitmq:
@@ -32,6 +41,13 @@ management:
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
  endpoints.web.exposure.include: prometheus, health
+  tracing:
+    enabled: ${TRACING_ENABLED:false}
+    sampling:
+      probability: ${TRACING_PROBABILITY:1.0}
+  otlp:
+    tracing:
+      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}


 storage:
@@ -0,0 +1,17 @@
+<configuration>
+
+    <springProperty scope="configuration" name="logType" source="logging.type"/>
+    <springProperty scope="context" name="application.name" source="spring.application.name"/>
+    <springProperty scope="context" name="version" source="project.version"/>
+    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="${logType}"/>
+    </root>
+
+</configuration>
@@ -46,12 +46,11 @@ public class BdrJsonBuildTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(File file) {

-        try (PDDocument pdDocument = Loader.loadPDF(file)) {
-            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
-                    pdDocument,
-                    new ImageServiceResponse(),
-                    new TableServiceResponse()));
-        }
+        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                file,
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                file.toString()));
    }


@@ -6,7 +6,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

-import org.apache.pdfbox.Loader;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -96,9 +95,10 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(pdfFileResource.getFile()),
+                pdfFileResource.getFile(),
                new ImageServiceResponse(),
-                new TableServiceResponse()));
+                new TableServiceResponse(),
+                filePath));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -0,0 +1,34 @@
+package com.knecon.fforesight.service.layoutparser.server;
+
+import java.util.Arrays;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class LayoutparserEnd2EndTest extends AbstractTest {
+
+    @Autowired
+    private LayoutParsingPipeline layoutParsingPipeline;
+
+
+    @Test
+    @SneakyThrows
+    public void testLayoutParserEndToEnd() {
+
+        prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
+        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
+    }
+
+}
@@ -6,6 +6,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;

+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -17,7 +18,9 @@ import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest
 import lombok.SneakyThrows;

 public class DocumentDataTests extends BuildDocumentTest {
+
    @Test
+    @Disabled // This test takes waaaaaay too long, it's ridiculous
    @SneakyThrows
    public void createDocumentDataForAllFiles() {

@@ -36,11 +39,12 @@ public class DocumentDataTests extends BuildDocumentTest {
        for (String pdfFileName : pdfFileNames) {
            System.out.println(pdfFileName);
            DocumentData documentData = DocumentDataMapper.toDocumentData(buildGraph(resource.getFile().toPath().getParent().relativize(Path.of(pdfFileName)).toString()));
-                        File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
+            File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
            outputFile.toPath().getParent().toFile().mkdirs();
            try (var out = new FileOutputStream(outputFile.toString().replace(".pdf", ".json"))) {
                ObjectMapperFactory.create().writeValue(out, documentData);
            }
        }
    }
+
 }
@@ -5,7 +5,6 @@ import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;

-import org.apache.pdfbox.Loader;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;
@@ -56,9 +55,10 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
    private void writeJsons(Path filename) {

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(filename.toFile()),
+                filename.toFile(),
                new ImageServiceResponse(),
-                new TableServiceResponse()));
+                new TableServiceResponse(),
+                filename.toFile().toString()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -47,7 +47,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentTest {
    @Disabled
    public void visualizeCraftedDocument() {

-        String filename = "files/crafted document.pdf";
+        String filename = "files/1 Abamectin_prr.pdf";
        visualizePdf(filename);
    }

@@ -0,0 +1,203 @@
+package com.knecon.fforesight.service.layoutparser.server.graph;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import lombok.SneakyThrows;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.cos.COSArray;
+import org.apache.pdfbox.cos.COSBase;
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSNumber;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureElement;
+import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureNode;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.text.PDFMarkedContentExtractor;
+import org.apache.pdfbox.text.TextPosition;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;
+
+/**
+ * @author mkl
+ */
+@Disabled
+public class ExtractMarkedContentTest {
+    final static File RESULT_FOLDER = new File("target/test-outputs", "extract");
+
+    @BeforeEach
+    public void setUpBeforeClass() throws Exception {
+        RESULT_FOLDER.mkdirs();
+    }
+
+    /**
+     * <a href="https://stackoverflow.com/questions/54956720/how-to-replace-a-space-with-a-word-while-extract-the-data-from-pdf-using-pdfbox">
+     * How to replace a space with a word while extract the data from PDF using PDFBox
+     * </a>
+     * <br/>
+     * <a href="https://drive.google.com/open?id=10ZkdPlGWzMJeahwnQPzE6V7s09d1nvwq">
+     * test.pdf
+     * </a> as "testWPhromma.pdf"
+     * <p>
+     * This test shows how to, in principle, extract tagged text.
+     * </p>
+     */
+    @Test
+    @SneakyThrows
+    public void testExtractTestWPhromma() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "testWPhromma.pdf");
+        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+            document.close();
+        }
+    }
+
+    /**
+     * <a href="https://stackoverflow.com/questions/59192443/get-tags-related-bboxs-even-though-there-is-no-attributes-a-in-document-cata">
+     * Get tag's related BBox's even though there is no attributes (/A in document catalog structure) related to Layout in PDFBox?
+     * </a>
+     * <br/>
+     * <a href="https://drive.google.com/file/d/1_-tuWuReaTvrDsqQwldTnPYrMHSpXIWp/view?usp=sharing">
+     * res_multipage.pdf
+     * </a>
+     * <p>
+     * This test shows how to, in principle, extract tagged text from this document.
+     * </p>
+     */
+    @Test
+    public void testExtractResMultipage() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "res_multipage.pdf");
+
+        try(PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+            document.close();
+        }
+    }
+
+    /**
+     * <a href="https://issues.apache.org/jira/browse/PDFBOX-5613">
+     * PDFBOX-5613 - uncorrent paragraph split
+     * </a>
+     * <br/>
+     * <a href="https://issues.apache.org/jira/secure/attachment/13058595/Daily%20Report.pdf">
+     * Daily Report.pdf
+     * </a>
+     * <p>
+     * Making use of the marked content, the contents of the file can better be extracted.
+     * </p>
+     */
+    @Test
+    public void testExtractDailyReport() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "Daily Report.pdf");
+        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+            document.close();
+        }
+    }
+
+    /**
+     * @see #testExtractTestWPhromma()
+     */
+    void showStructure(PDStructureNode node, Map<PDPage, Map<Integer, PDMarkedContent>> markedContents) {
+        String structType = null;
+        PDPage page = null;
+        if (node instanceof PDStructureElement) {
+            PDStructureElement element = (PDStructureElement) node;
+            structType = element.getStructureType();
+            page = element.getPage();
+        }
+        Map<Integer, PDMarkedContent> theseMarkedContents = markedContents.get(page);
+        System.out.printf("<%s>\n", structType);
+        for (Object object : node.getKids()) {
+            if (object instanceof COSArray) {
+                for (COSBase base : (COSArray) object) {
+                    if (base instanceof COSDictionary) {
+                        showStructure(PDStructureNode.create((COSDictionary) base), markedContents);
+                    } else if (base instanceof COSNumber) {
+                        showContent(((COSNumber)base).intValue(), theseMarkedContents);
+                    } else {
+                        System.out.printf("?%s\n", base);
+                    }
+                }
+            } else if (object instanceof PDStructureNode) {
+                showStructure((PDStructureNode) object, markedContents);
+            } else if (object instanceof Integer) {
+                showContent((Integer)object, theseMarkedContents);
+            } else {
+                System.out.printf("?%s\n", object);
+            }
+
+        }
+        System.out.printf("</%s>\n", structType);
+    }
+
+    /**
+     * @see #showStructure(PDStructureNode, Map)
+     * @see #testExtractTestWPhromma()
+     */
+    void showContent(int mcid, Map<Integer, PDMarkedContent> theseMarkedContents) {
+        PDMarkedContent markedContent = theseMarkedContents != null ? theseMarkedContents.get(mcid) : null;
+        List<Object> contents = markedContent != null ? markedContent.getContents() : Collections.emptyList();
+        StringBuilder textContent =  new StringBuilder();
+        for (Object object : contents) {
+            if (object instanceof TextPosition) {
+                textContent.append(((TextPosition)object).getUnicode());
+            } else {
+                textContent.append("?" + object);
+            }
+        }
+        System.out.printf("%s\n", textContent);
+    }
+}
@@ -1,10 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

-import java.io.FileOutputStream;
+import java.io.File;
 import java.nio.file.Path;

-import org.apache.pdfbox.Loader;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -19,18 +17,16 @@ import lombok.SneakyThrows;
 public class ViewerDocumentTest extends BuildDocumentTest {

    @Test
-    @Disabled
    @SneakyThrows
    public void testViewerDocument() {

-        LayoutGridService layoutGridService = new LayoutGridService();
-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
-        String fileName = "files/new/VV-511309_OCR.pdf";
-        Document document = buildGraph(fileName, LayoutParsingType.DOCUMINE);
+        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
-        try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getFile()); var out = new FileOutputStream(tmpFileName)) {
-            viewerDocumentService.createViewerDocument(pdDocument, document, out, true);
-        }
+        LayoutGridService layoutGridService = new LayoutGridService();
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
+        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
+        var documentFile = new ClassPathResource(fileName).getFile();
+        viewerDocumentService.createViewerDocument(documentFile, document, new File(tmpFileName), true);
    }

 }
@@ -27,14 +27,14 @@ public class TextPositionSequenceTest {
            + "  \"pageWidth\": 600\n"  //
            + "}";

-    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper(), new StorageProperties());
+    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper());


    @Test
    @SneakyThrows
    public void testDeserializationWithJackson() {

-        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithJackson(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+        TextPositionSequence textPositionSequence = objectSerializer.deserialize(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
                TextPositionSequence.class);

        assertPropertiesAfterJsonDeserialization(textPositionSequence);
@@ -1,5 +1,26 @@
 package com.knecon.fforesight.service.layoutparser.server.segmentation;

+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.awt.geom.Rectangle2D;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -15,19 +36,8 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+
 import lombok.SneakyThrows;
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
-import java.awt.geom.Rectangle2D;
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.assertj.core.api.Assertions.assertThat;

 public class PdfSegmentationServiceTest extends AbstractTest {

@@ -50,12 +60,14 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    private SectionsBuilderService sectionsBuilderService;


-    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+    @SneakyThrows
+    public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {

-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                originDocument,
                new ImageServiceResponse(),
-                new TableServiceResponse());
+                tableServiceResponse,
+                "document");

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -65,6 +77,58 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @SneakyThrows
+    public ClassificationDocument buildClassificationDocument(File originDocument) {
+
+        return buildClassificationDocument(originDocument, new TableServiceResponse());
+    }
+
+
+    @Test
+    public void tablesToHtmlDebugger() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");
+
+    }
+
+
+    @Test
+    public void tablesToHtmlDebuggerWithCVResponse() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
+        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
+        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
+
+        toHtml(document, "/tmp/ScanRotationBorder.html");
+    }
+
+
+    @Test
+    public void testScanRotationBorderIsIgnored() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
+        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
+        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+
+        // Quality of the table parsing is not good, because the file is rotated at scanning.
+        // We only asset that the table border is not the page border.
+        tables.forEach(table -> {
+            assertThat(table.getMinX()).isGreaterThan(75);
+            assertThat(table.getMaxX()).isLessThan(512);
+        });
+    }
+
+
    @Test
    @SneakyThrows
    public void testMapping() {
@@ -92,7 +156,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(table.getColCount()).isEqualTo(6);
@@ -106,7 +170,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -124,7 +188,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(9);
@@ -142,7 +206,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -155,19 +219,36 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


-    @Test
+    @Test // Non-sense test
    public void testDoc56Page170() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 4);

        validateTable(document, 0, 1, 1, 0, 0);
        validateTable(document, 1, 2, 2, 0, 0);
-        validateTable(document, 2, 7, 20, 0, 140);
-        validateTable(document, 3, 8, 31, 0, 170);
+        validateTable(document, 2, 6, 20, 0, 0);
+        validateTable(document, 3, 7, 31, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc211() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/211.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 5, 4, 0, 0);
+        validateTable(document, 1, 5, 15, 14, 0);
+        validateTable(document, 2, 5, 14, 11, 0);
+        validateTable(document, 3, 5, 3, 0, 0);

    }

@@ -177,11 +258,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

-        validateTable(document, 0, 8, 8, 0, 2);
+        validateTable(document, 0, 8, 8, 0, 0);

        List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
                        "Author, date",
@@ -191,18 +272,18 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                        "Method meets analytical validation criteria",
                        "Remarks (in case validation criteria are not met)",
                        "Acceptability of the method"),
-                Arrays.asList("",
+                Arrays.asList("Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
                        "Evans P.G. 2001 TMJ4569B, VV-323245",
                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
-                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
                        "Y",
                        "N/A",
@@ -218,7 +299,9 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        toHtml(document, "/tmp/html.html");

        validateTableSize(document, 4);

@@ -231,17 +314,29 @@ public class PdfSegmentationServiceTest extends AbstractTest {


    @Test
+    @Disabled // FIXME Fake Redactions leads to more cells, no solution for this currently
    public void testDocA20622APartB9Page185() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

-        validateTableSize(document, 2);
+        validateTableSize(document, 1);

-        validateTable(document, 0, 5, 5, 0, 23);
-        validateTable(document, 1, 11, 9, 0, 36);
+        validateTable(document, 0, 7, 4, 0, 0);
+    }

+
+    @Test
+    public void testDocA20622APartB9Page185FixedDoc() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185_fixed.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 7, 4, 0, 0);
    }


@@ -250,7 +345,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izZRMS (CZ) fRR Part B7_Page123.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 6);

@@ -269,7 +364,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04_Page11.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 3);

@@ -285,7 +380,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10_Page532.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 9, 0, 0);
@@ -298,7 +393,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

@@ -312,7 +407,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 6, 7, 0);
@@ -325,10 +420,10 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);
-        validateTable(document, 0, 10, 6, 0, 1);
+        validateTable(document, 0, 10, 6, 0, 0);

    }

@@ -338,7 +433,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);
        validateTable(document, 0, 2, 2, 0, 0);
@@ -353,7 +448,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

@@ -369,7 +464,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

@@ -384,7 +479,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

@@ -399,7 +494,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

@@ -414,7 +509,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 4);

@@ -431,7 +526,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

@@ -446,12 +541,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

-        validateTable(document, 0, 6, 8, 0, 2);
-        validateTable(document, 1, 6, 8, 0, 1);
+        validateTable(document, 0, 6, 8, 0, 0);
+        validateTable(document, 1, 6, 8, 0, 0);

    }

@@ -461,7 +556,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

@@ -475,7 +570,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

@@ -484,12 +579,37 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @SneakyThrows
+    private void toHtml(ClassificationDocument document, String filename) {
+
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+        StringBuilder sb = new StringBuilder();
+
+        int currentPage = 1;
+        for (var table : tables) {
+            if (currentPage != table.getPage()) {
+                currentPage = table.getPage();
+                sb.append("---------------------- Page ").append(currentPage).append("--------------\n");
+            }
+            sb.append("\n\n");
+            sb.append(table.getTextAsHtml());
+        }
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(Path.of(filename).toFile())) {
+            fileOutputStream.write(sb.toString().getBytes());
+        }
+    }
+
+
    private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {

        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
        List<List<Cell>> rows = table.getRows();
        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();

+        for (List<Cell> row : table.getRows()) {
+            row.forEach(r -> System.out.println(r.toString()));
+        }
        assertThat(emptyCellsFoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);

        assertThat(table.getColCount()).isEqualTo(colCount);
@@ -1,21 +1,39 @@
 package com.knecon.fforesight.service.layoutparser.server.services;

+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;

+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;

-public class RulingCleaningServiceTest {
+public class RulingCleaningServiceTest extends BuildDocumentTest {

    @Test
 //    @Disabled
@@ -25,13 +43,98 @@ public class RulingCleaningServiceTest {
        String fileName = "files/211.pdf";
        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
-        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
-
        RulingCleaningService rulingCleaningService = new RulingCleaningService();
+        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
        List<CleanRulings> cleanRulingsPerPage = new LinkedList<>();
        for (PageContents pageContent : pageContents) {
-            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings(), 8, 20));
+            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings()));
+        }
+
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testTableExtraction() {
+
+        LayoutGridService layoutGridService = new LayoutGridService();
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
+
+        ClassPathResource resource = new ClassPathResource("files");
+        List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
+                .filter(path -> path.getFileName().toString().endsWith(".pdf"))
+                .map(Path::toAbsolutePath)
+                .map(Path::toString)
+                .toList();
+
+        for (int i = 0; i < pdfFileNames.size(); i++) {
+            writeJsons(Path.of(pdfFileNames.get(i)));
        }
    }

+
+    @SneakyThrows
+    private void writeJsons(Path filename) {
+
+        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                filename.toFile(),
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                filename.toFile().toString()));
+        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                filename.toFile(),
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                filename.toFile().toString()));
+        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
+        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
+        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
+            String tmpFileNameBefore = "C:/Users/YANNIK~1/AppData/Local/Temp/before." + filename.getFileName().toString();
+            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
+                PdfDraw.drawDocumentGraph(pdDocument, documentGraphBefore);
+                pdDocument.save(tmpFileNameBefore);
+            }
+            String tmpFileNameAfter = "C:/Users/YANNIK~1/AppData/Local/Temp/after." + filename.getFileName().toString();
+            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
+                PdfDraw.drawDocumentGraph(pdDocument, documentGraphAfter);
+                pdDocument.save(tmpFileNameAfter);
+
+            }
+        }
+    }
+
+
+    @SneakyThrows
+    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2, String pdfName) {
+
+        List listStructure1 = structure1.streamAllEntries()
+                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
+                .map(DocumentStructure.EntryData::getProperties)
+                .map(properties -> {
+                    var builder = Table.builder();
+                    PropertiesMapper.parseTableProperties(properties, builder);
+                    return builder.build();
+                })
+                .toList();
+
+        List listStructure2 = structure2.streamAllEntries()
+                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
+                .map(DocumentStructure.EntryData::getProperties)
+                .map(properties -> {
+                    var builder = Table.builder();
+                    PropertiesMapper.parseTableProperties(properties, builder);
+                    return builder.build();
+                })
+                .toList();
+
+        for (int i = 0; i < listStructure1.size(); i++) {
+            Table tableNode1 = (Table) listStructure1.get(i);
+            Table tableNode2 = (Table) listStructure2.get(i);
+            if (tableNode1.getNumberOfRows() != tableNode2.getNumberOfRows() || tableNode1.getNumberOfCols() != tableNode2.getNumberOfCols()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
 }
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

 import java.io.InputStream;
+import java.util.Map;
 import java.util.Optional;

 import org.junit.jupiter.api.AfterEach;
@@ -12,13 +13,17 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;

+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
@@ -53,7 +58,6 @@ public abstract class AbstractTest {
    protected final static String POSITION_FILE_ID = "positions";
    protected final static String PAGES_FILE_ID = "pages";
    protected final static String TENANT_ID = "tenant";
-    protected final static String SECTION_GRID_ID = "section";
    protected final static String VIEWER_DOCUMENT_ID = "viewer";
    protected final static String SIMPLIFIED_ID = "simplified";

@@ -70,7 +74,6 @@ public abstract class AbstractTest {
                .positionBlockFileStorageId(POSITION_FILE_ID)
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
-                .sectionGridStorageId(SECTION_GRID_ID)
                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }
@@ -101,8 +104,15 @@ public abstract class AbstractTest {
    protected LayoutParsingRequest prepareStorage(InputStream fileInputStream) {

        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileInputStream);
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+    }
+
+
+    protected LayoutParsingRequest buildDefaultLayoutParsingRequest(LayoutParsingType layoutParsingType) {
+
        return LayoutParsingRequest.builder()
-                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
+                .identifier(Map.of("fileId", "1337"))
+                .layoutParsingType(layoutParsingType)
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
@@ -111,7 +121,6 @@ public abstract class AbstractTest {
                .positionBlockFileStorageId(POSITION_FILE_ID)
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
-                .sectionGridStorageId(SECTION_GRID_ID)
                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }
@@ -135,19 +144,7 @@ public abstract class AbstractTest {
        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);

-        return LayoutParsingRequest.builder()
-                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
-                .originFileStorageId(ORIGIN_FILE_ID)
-                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
-                .structureFileStorageId(STRUCTURE_FILE_ID)
-                .textBlockFileStorageId(TEXT_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
-                .pageFileStorageId(PAGES_FILE_ID)
-                .simplifiedTextStorageId(SIMPLIFIED_ID)
-                .sectionGridStorageId(SECTION_GRID_ID)
-                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
-                .build();
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
    }


@@ -165,6 +162,13 @@ public abstract class AbstractTest {
    @ComponentScan("com.knecon.fforesight.service.layoutparser")
    public static class TestConfiguration {

+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
+        }
+
    }

 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

-import java.io.InputStream;
+import java.io.File;

-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

@@ -25,11 +23,9 @@ public abstract class BuildDocumentTest extends AbstractTest {
    @SneakyThrows
    protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {

-        ClassPathResource fileResource = new ClassPathResource(filename);
+        File fileResource = new ClassPathResource(filename).getFile();
        prepareStorage(filename);
-        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream.readAllBytes())) {
-            return layoutParsingPipeline.parseLayout(layoutParsingType, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse());
-        }
+        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), filename);
    }


@@ -1,124 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server.utils;
-
-import static java.io.File.createTempFile;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.commons.io.IOUtils;
-import org.springframework.stereotype.Service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
-@Service
-public class FileSystemBackedStorageService implements StorageService {
-
-    private final Map<String, File> dataMap = new HashMap<>();
-
-
-    public FileSystemBackedStorageService() {
-
-    }
-
-
-    @SneakyThrows
-    @Override
-    public void downloadTo(String tenantId, String objectId, File destinationFile) {
-
-        var res = dataMap.get(objectId);
-        if (res == null) {
-            throw new StorageObjectDoesNotExist(new RuntimeException());
-        }
-
-        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
-    }
-
-
-    @Override
-    public void deleteObject(String tenantId, String objectId) {
-
-        dataMap.remove(objectId);
-    }
-
-
-    @Override
-    public boolean objectExists(String tenantId, String objectId) {
-
-        return dataMap.containsKey(objectId);
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
-
-        File tempFile = createTempFile("test", ".tmp");
-        getMapper().writeValue(new FileOutputStream(tempFile), any);
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    private ObjectMapper getMapper() {
-
-        return ObjectMapperFactory.create();
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
-
-        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
-            throw new StorageObjectDoesNotExist("Stored object not found");
-        }
-        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
-    }
-
-
-    public List<String> listPaths() {
-
-        return new ArrayList<>(dataMap.keySet());
-    }
-
-
-    public List<String> listFilePaths() {
-
-        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
-    }
-
-
-    @Override
-    @SneakyThrows
-    public void storeObject(String tenantId, String objectId, InputStream stream) {
-
-        File tempFile = createTempFile("test", ".tmp");
-
-        try (var fileOutputStream = new FileOutputStream(tempFile)) {
-            IOUtils.copy(stream, fileOutputStream);
-        }
-
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    public void clearStorage() {
-
-        this.dataMap.forEach((k, v) -> {
-            v.delete();
-        });
-        this.dataMap.clear();
-    }
-
-}
@@ -7,6 +7,9 @@ fforesight.tenants.remote: true
 server:
  port: 8080

+logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
+logging.type: ${LOGGING_TYPE:CONSOLE}
+
 spring:
  main:
    allow-circular-references: true # FIXME
@@ -1,16 +0,0 @@
-<Configuration>
-
-    <Appenders>
-        <Console name="CONSOLE" target="SYSTEM_OUT">
-            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
-        </Console>
-    </Appenders>
-
-    <Loggers>
-        <Root level="warn">
-            <AppenderRef ref="CONSOLE"/>
-        </Root>
-        <Logger name="com.iqser" level="info"/>
-    </Loggers>
-
-</Configuration>
@@ -0,0 +1,17 @@
+<configuration>
+
+    <springProperty scope="configuration" name="logType" source="logging.type"/>
+    <springProperty scope="context" name="application.name" source="spring.application.name"/>
+    <springProperty scope="context" name="version" source="project.version"/>
+    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="${logType}"/>
+    </root>
+
+</configuration>
@@ -2,7 +2,14 @@
 dir=${PWD##*/}
 gradle assemble

-buildNumber=${1:-1}
+# Get the current Git branch
+branch=$(git rev-parse --abbrev-ref HEAD)

-gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$USER-$buildNumber
-echo "nexus.knecon.com:5001/red/${dir}-server-v1:$USER-$buildNumber"
+# Get the short commit hash (first 5 characters)
+commit_hash=$(git rev-parse --short=5 HEAD)
+
+# Combine branch and commit hash
+buildName="${USER}-${branch}-${commit_hash}"
+
+gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName --no-build-cache
+echo "nexus.knecon.com:5001/ff/${dir}-service-server:$buildName"
Author	SHA1	Message	Date
Dominique Eifländer	9e854379e7	Merge branch 'RED-1137' into 'main' RED-1137: Do not observe actuator endpoints See merge request fforesight/layout-parser!92	2023-12-20 14:11:31 +01:00
Dominique Eifländer	b779c72041	RED-1137: Do not observe actuator endpoints	2023-12-20 14:05:00 +01:00
Dominique Eifländer	760a809900	Merge branch 'RED-7384' into 'main' RED-7384: fixes for migration See merge request fforesight/layout-parser!91	2023-12-20 12:40:00 +01:00
Kilian Schüttler	ba1c7c07ab	RED-7384: fixes for migration	2023-12-20 12:40:00 +01:00
Dominique Eifländer	ca0cbbcb49	Merge branch 'RED-5223' into 'main' RED-5223: Use tracing-commons from fforesight See merge request fforesight/layout-parser!90	2023-12-13 16:05:01 +01:00
Dominique Eifländer	da2cdc288e	RED-5223: Use tracing-commons from fforesight	2023-12-13 15:31:26 +01:00
Dominique Eifländer	68da328889	Merge branch 'queueHotfix' into 'main' hotfix: removed dlq from response queue to be equal to persistence-service See merge request fforesight/layout-parser!89	2023-12-13 09:54:43 +01:00
Dominique Eifländer	711548d1a7	hotfix: removed dlq from response queue to be equal to persistence-service	2023-12-13 09:47:27 +01:00
Dominique Eifländer	2bddcdafee	Merge branch 'RED-5223' into 'main' RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs See merge request fforesight/layout-parser!88	2023-12-11 15:13:22 +01:00
Dominique Eifländer	750ccf4ce2	RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs	2023-12-11 15:06:23 +01:00
Ali Oezyetimoglu	57b5d3f48e	Merge branch 'RED-7714' into 'main' RED-7715 - Add log4j config to enable switching between json/line logs See merge request fforesight/layout-parser!87	2023-12-06 13:00:28 +01:00
Andrei Isvoran	d8c9659469	RED-7715 - Add log4j config to enable switching between json/line logs	2023-12-06 11:59:42 +02:00
Kilian Schüttler	30f060e36c	Merge branch 'enable-caching' into 'main' enable caching in build See merge request fforesight/layout-parser!84	2023-11-24 10:36:55 +01:00
Kilian Schuettler	53a5824e6c	enable caching in build	2023-11-24 10:24:50 +01:00
Dominique Eifländer	e2bcf971c9	Merge branch 'DM-589' into 'main' DM-589: Filter wrong detected cells that borders from rotation at scanning See merge request fforesight/layout-parser!83	2023-11-20 16:07:27 +01:00
Dominique Eifländer	dacc2f7f43	DM-589: Filter wrong detected cells that borders from rotation at scanning	2023-11-20 15:54:02 +01:00
Dominique Eifländer	144a9591a2	Merge branch 'TAAS-103-hotfix' into 'main' * added back in if statement See merge request fforesight/layout-parser!82	2023-11-16 12:48:48 +01:00
yhampe	207d9dec97	* added back in if statement * removed not needed commentar	2023-11-16 12:40:49 +01:00
Yannik Hampe	09ee90222e	Merge branch 'TAAS-103' into 'main' TAAS-103: Table Detection and rotated text See merge request fforesight/layout-parser!81	2023-11-16 09:13:41 +01:00
yhampe	1316a067fe	* removed double chechking for height of cell	2023-11-16 08:51:12 +01:00
yhampe	e203210ade	* removed not needed properties	2023-11-16 08:23:58 +01:00
yhampe	b25d46291a	* checkstyle	2023-11-16 08:12:47 +01:00
yhampe	84148d3b6e	* fixed tests	2023-11-16 07:51:08 +01:00
Dominique Eifländer	a6ba66b1aa	TAAS-103: Fixed values in wrong cells	2023-11-15 13:36:46 +01:00
yhampe	c3e69b2cdf	* fixed bug with incorrect empty cell count by adding threshhold to cell.contains	2023-11-15 10:44:47 +01:00
yhampe	f69331e7d8	*renamed page to firstPage in DocumentStructure and Table	2023-11-07 10:21:19 +01:00
yhampe	01493dc033	TAAS-103: Table Detection and rotated text * added page property to DocumentStructure to be able to get page of found tables * added a method to TableExtractionService to get the table area * added calculateMinCharWidthAndMaxCharHeightInsideTable to LayoutParsingPipeline to calculate the values based upon table area * refactored PDFLinesTextStripper for better readability *removed textMatrix from RedTextPosition as it is no longer needed	2023-11-07 08:47:28 +01:00
yhampe	459e0c8be7	TAAS-103:	2023-11-07 08:39:15 +01:00
Kilian Schüttler	1b1f777706	Merge branch 'RED-7806' into 'main' RED-7806 - Specific customer document cannot be processed See merge request fforesight/layout-parser!79	2023-10-25 10:50:34 +02:00
Corina Olariu	0e0a811f9d	RED-7806 - Specific customer document cannot be processed - add brackets	2023-10-25 11:36:54 +03:00
Corina Olariu	efa3d75479	RED-7806 - Specific customer document cannot be processed - check for font name null before using to avoid the NPE	2023-10-25 09:16:47 +03:00
Kilian Schüttler	9abdc6d44d	Merge branch 'RED-7434' into 'main' RED-7434 - Remove Section Grid entirely See merge request fforesight/layout-parser!78	2023-10-20 10:07:01 +02:00
Corina Olariu	3bab61c446	RED-7434 - Remove Section Grid entirely - remove sectionGrid relation (including SectionGridCreatorService) - update junit tests	2023-10-20 09:09:22 +03:00
Dominique Eifländer	d17517d3c3	Merge branch 'hotfix-bdr-doc' into 'main' hotfix: Fixed parsing for specific taas document See merge request fforesight/layout-parser!77	2023-10-18 16:12:15 +02:00
Dominique Eifländer	567cbc178b	hotfix: Fixed parsing for specific taas document	2023-10-17 15:52:19 +02:00
Kilian Schüttler	3c53772765	Merge branch 'RED-7759' into 'main' RED-7759: Upgraded storage-commons to newest windwos compatible version See merge request fforesight/layout-parser!76	2023-10-13 12:21:04 +02:00
Dominique Eifländer	8647cf5a18	RED-7759: Upgraded storage-commons to newest windwos compatible version	2023-10-13 12:15:22 +02:00
Kilian Schüttler	310c07b200	Merge branch 'RED-7607-WIP' into 'main' RED-7607 - Rotating pages leads to lost annotations (RM & DM) See merge request fforesight/layout-parser!75	2023-10-05 13:34:12 +02:00
Corina Olariu	daba0bf8a6	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - remove finally clause	2023-10-04 17:46:46 +03:00
Corina Olariu	3839de215c	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - rollback to getDir().getDegrees()	2023-10-04 15:27:13 +03:00
Corina Olariu	b4d68594f1	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - use rotation instead of getDir().getDegrees()	2023-10-04 14:22:15 +03:00
Corina Olariu	99ed331a1e	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - use getXDirAdj instead of getX - add fontSizeCounter for landscape pages also	2023-10-04 14:13:38 +03:00
Corina Olariu	f2c0991987	RED-7607 - Rotating pages leads to lost annotations (RM & DM) - fix PMD findings	2023-10-04 14:09:46 +03:00
Timo Bejan	b8ef55e6e2	Merge branch 'TAAS-104' into 'main' TAAS-104: merge visually intersecting Paragraphs See merge request fforesight/layout-parser!73	2023-09-05 17:08:40 +02:00
Kilian Schuettler	5792ff4a93	TAAS-104: merge visually intersecting Paragraphs * fix build	2023-09-05 16:54:23 +02:00
Kilian Schuettler	621c3f269d	TAAS-104: merge visually intersecting Paragraphs	2023-09-05 16:09:05 +02:00
Dominique Eifländer	8dba392904	Merge branch 'RED-7461' into 'main' RED-7461: Fixed wrong textblock classifation if footer is marked as header See merge request fforesight/layout-parser!72	2023-09-01 12:14:38 +02:00
deiflaender	306a53ea79	RED-7461: Fixed wrong textblock classifation if footer is marked as header	2023-09-01 12:07:47 +02:00
Kilian Schüttler	754fd8f933	Merge branch 'TAAS-89' into 'main' TAAS-89: added log entry and an end2end test See merge request fforesight/layout-parser!71	2023-08-31 14:40:48 +02:00
Kilian Schuettler	28ec4c9ccb	TAAS-89: added log entry and an end2end test	2023-08-31 14:28:18 +02:00