TAAS-104: merge visually intersecting Paragraphs

* fix static import
2023-09-06 11:38:40 +02:00
57 changed files with 698 additions and 947 deletions
@@ -10,7 +10,7 @@ deploy:
  script:
    - echo "Building with gradle version ${BUILDVERSION}"
    - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
  artifacts:
    reports:
@@ -24,8 +24,6 @@ tasks.named<Test>("test") {
    reports {
        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
    }
-    minHeapSize = "512m"
-    maxHeapSize = "2048m"
 }

 tasks.test {
@@ -36,6 +36,9 @@ public record LayoutParsingRequest(
        @Schema(description = "Path where the Simplified Text File will be stored.")//
        @NonNull String simplifiedTextStorageId,//
        @Schema(description = "Path where the Viewer Document PDF will be stored.")//
-        @NonNull String viewerDocumentStorageId) {
+        @NonNull String viewerDocumentStorageId,//
+        @Deprecated//
+        @Schema(description = "Path where the Section Grid will be stored.")//
+        @NonNull String sectionGridStorageId) {

 }
@@ -15,8 +15,8 @@ dependencies {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
-    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("com.iqser.red.commons:storage-commons:2.40.0")

    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -3,16 +3,13 @@ package com.knecon.fforesight.service.layoutparser.processor;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
-import java.io.File;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;

-import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
@@ -38,6 +35,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
@@ -54,121 +52,93 @@ import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDF
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;

-import io.micrometer.observation.Observation;
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.annotation.Observed;
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

-@SuppressWarnings("PMD.CloseResource")
@Slf4j
@Service
@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class LayoutParsingPipeline {

-    ImageServiceResponseAdapter imageServiceResponseAdapter;
-    CvTableParsingAdapter cvTableParsingAdapter;
-    LayoutParsingStorageService layoutParsingStorageService;
-    SectionsBuilderService sectionsBuilderService;
-    TaasClassificationService taasClassificationService;
-    RedactManagerClassificationService redactManagerClassificationService;
-    DocuMineClassificationService docuMineClassificationService;
-    SimplifiedSectionTextService simplifiedSectionTextService;
-    BodyTextFrameService bodyTextFrameService;
-    RulingCleaningService rulingCleaningService;
-    TableExtractionService tableExtractionService;
-    TaasBlockificationService taasBlockificationService;
-    DocuMineBlockificationService docuMineBlockificationService;
-    RedactManagerBlockificationService redactManagerBlockificationService;
-    ViewerDocumentService viewerDocumentService;
-    ObservationRegistry observationRegistry;
+    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
+    private final CvTableParsingAdapter cvTableParsingAdapter;
+    private final LayoutParsingStorageService layoutParsingStorageService;
+    private final SectionsBuilderService sectionsBuilderService;
+    private final SectionGridCreatorService sectionGridCreatorService;
+    private final TaasClassificationService taasClassificationService;
+    private final RedactManagerClassificationService redactManagerClassificationService;
+    private final DocuMineClassificationService docuMineClassificationService;
+    private final SimplifiedSectionTextService simplifiedSectionTextService;
+    private final BodyTextFrameService bodyTextFrameService;
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;
+    private final ViewerDocumentService viewerDocumentService;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {

        long start = System.currentTimeMillis();
-        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

-        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-        File viewerDocumentFile = File.createTempFile("viewer_document", ".pdf");
+        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
+            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            }

-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            TableServiceResponse tableServiceResponse = new TableServiceResponse();
+            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+            }
+
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+
+            int numberOfPages = originDocument.getNumberOfPages();
+
+            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
+            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+            try (var out = new ByteArrayOutputStream()) {
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out, false);
+                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
+            }
+
+            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+            }
+
+            return LayoutParsingFinishedEvent.builder()
+                    .identifier(layoutParsingRequest.identifier())
+                    .numberOfPages(numberOfPages)
+                    .duration(System.currentTimeMillis() - start)
+                    .message(format("""
+                                    Layout parsing has finished in %.02f s.
+                                    identifiers: %s
+                                    %s
+                                    Files have been saved with Ids:
+                                    Structure: %s
+                                    Text: %s
+                                    Positions: %s
+                                    PageData: %s
+                                    Simplified Text: %s
+                                    Viewer Doc: %s""",
+                            ((float) (System.currentTimeMillis() - start)) / 1000,
+                            layoutParsingRequest.identifier(),
+                            buildSemanticNodeCountMessage(numberOfPages, documentGraph.buildSemanticNodeCounts()),
+                            layoutParsingRequest.structureFileStorageId(),
+                            layoutParsingRequest.textBlockFileStorageId(),
+                            layoutParsingRequest.positionBlockFileStorageId(),
+                            layoutParsingRequest.pageFileStorageId(),
+                            layoutParsingRequest.simplifiedTextStorageId(),
+                            layoutParsingRequest.viewerDocumentStorageId()))
+                    .build();
        }
-
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
-        }
-
-        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
-                originFile,
-                imageServiceResponse,
-                tableServiceResponse,
-                layoutParsingRequest.identifier().toString());
-        log.info("Building document graph for {}", layoutParsingRequest.identifier());
-
-        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
-
-        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
-
-        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
-        viewerDocumentService.createViewerDocument(originFile, documentGraph, viewerDocumentFile, false);
-        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
-            log.info("Building research document data for {}", layoutParsingRequest.identifier());
-            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
-            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
-        }
-
-        originFile.delete();
-        viewerDocumentFile.delete();
-
-        return LayoutParsingFinishedEvent.builder()
-                .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(documentGraph.getNumberOfPages())
-                .duration(System.currentTimeMillis() - start)
-                .message(format("""
-                                Layout parsing has finished in %.02f s.
-                                identifiers: %s
-                                %s
-                                Files have been saved with Ids:
-                                Structure: %s
-                                Text: %s
-                                Positions: %s
-                                PageData: %s
-                                Simplified Text: %s
-                                Viewer Doc: %s""",
-                        ((float) (System.currentTimeMillis() - start)) / 1000,
-                        layoutParsingRequest.identifier(),
-                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId(),
-                        layoutParsingRequest.simplifiedTextStorageId(),
-                        layoutParsingRequest.viewerDocumentStorageId()))
-                .build();
-
-    }
-
-
-    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
-
-        AtomicReference<Document> documentReference = new AtomicReference<>();
-
-        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
-            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
-        });
-
-        return documentReference.get();
    }


@@ -176,47 +146,32 @@ public class LayoutParsingPipeline {

        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
                numberOfPages,
-                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
-                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
-                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
-                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
-                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
-                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
-                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
+                semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER));
    }


    @SneakyThrows
-    @Observed(name = "LayoutParsingPipeline", contextualName = "parse-layout")
    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
-                                              File originFile,
+                                              PDDocument originDocument,
                                              ImageServiceResponse imageServiceResponse,
-                                              TableServiceResponse tableServiceResponse,
-                                              String identifier) {
+                                              TableServiceResponse tableServiceResponse) {

-        PDDocument originDocument = openDocument(originFile);
-        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);

        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

+        originDocument.setAllSecurityToBeRemoved(true);
        long pageCount = originDocument.getNumberOfPages();

        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-
-            if (pageNumber % 100 == 0) {
-                // re-open document every once in a while to save on RAM. This has no significant performance impact.
-                // This is due to PDFBox caching all images and some other stuff with Soft References. This dereferences them and forces the freeing of memory.
-                originDocument.close();
-                originDocument = openDocument(originFile);
-            }
-
-            if (pageNumber % 100 == 0 || pageNumber == pageCount || pageNumber == 1) {
-                log.info("Extracting text on Page {} for {}", pageNumber, identifier);
-            }
-
            classificationDocument.setPages(classificationPages);
            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
            PDPage pdPage = originDocument.getPage(pageNumber - 1);
@@ -235,7 +190,10 @@ public class LayoutParsingPipeline {
            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);

            PDRectangle cropbox = pdPage.getCropBox();
-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                    stripper.getRulings(),
+                    stripper.getMinCharWidth(),
+                    stripper.getMaxCharHeight());

            ClassificationPage classificationPage = switch (layoutParsingType) {
                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
@@ -258,50 +216,28 @@ public class LayoutParsingPipeline {
                imageServiceResponseAdapter.findOcr(classificationPage);
            }

-            tableExtractionService.extractTables(cleanRulings, classificationPage);
-
+            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
            buildPageStatistics(classificationPage);
            increaseDocumentStatistics(classificationPage, classificationDocument);

            classificationPages.add(classificationPage);
        }

-        originDocument.close();
-
-        log.info("Calculating BodyTextFrame for {}", identifier);
        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
-        log.info("Classify TextBlocks for {}", identifier);
+
        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }

-        log.info("Building Sections for {}", identifier);
        sectionsBuilderService.buildSections(classificationDocument);
        sectionsBuilderService.addImagesToSections(classificationDocument);
+
        return classificationDocument;
    }


-    private void addNumberOfPagesToTrace(int numberOfPages, long size) {
-
-        if (observationRegistry.getCurrentObservation() != null) {
-            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
-            observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileSize", String.valueOf(size));
-        }
-    }
-
-
-    @SneakyThrows
-    private PDDocument openDocument(File originFile) {
-
-        PDDocument document = Loader.loadPDF(originFile);
-        document.setAllSecurityToBeRemoved(true);
-        return document;
-    }
-
-
    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {

        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor;

+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -9,11 +11,13 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;

+import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
@@ -23,7 +27,6 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

-import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -37,18 +40,16 @@ public class LayoutParsingStorageService {
    private final ObjectMapper objectMapper;


-    public PDDocument getOriginDocument(String storageId) throws IOException {
+    public PDDocument getOriginFile(String storageId) throws IOException {

-        return Loader.loadPDF(getOriginFile(storageId));
-    }
-
-
-    @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
-    public File getOriginFile(String storageId) throws IOException {
-
-        File tempFile = createTempFile("document", ".pdf");
-        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
-        return tempFile;
+        try (var originDocumentInputStream = getObject(storageId)) {
+            File tempFile = createTempFile("document", ".pdf");
+            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
+                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
+            }
+            return Loader.loadPDF(tempFile);
+        }
    }


@@ -74,7 +75,6 @@ public class LayoutParsingStorageService {
    }


-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
@@ -84,6 +84,12 @@ public class LayoutParsingStorageService {
    }


+    public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid);
+    }
+
+
    public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.researchDocumentStorageId(), researchDocumentData);
@@ -115,7 +121,6 @@ public class LayoutParsingStorageService {
    }


-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-simplified-text")
    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
@@ -133,10 +138,9 @@ public class LayoutParsingStorageService {


    @SneakyThrows
-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-viewer-document")
-    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, File out) {
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {

-        try (var in = new FileInputStream(out)) {
+        try (var in = new ByteArrayInputStream(out.toByteArray())) {

            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
        }
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.ArrayList;
 import java.util.List;

+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText;

@@ -24,6 +25,7 @@ public class ClassificationDocument {
    private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
    private boolean headlines;

+    private SectionGrid sectionGrid = new SectionGrid();
    private long rulesVersion;

 }
@@ -10,6 +10,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.amazonaws.services.kms.model.NotFoundException;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -83,7 +84,7 @@ public class Document implements GenericSemanticNode {
    @Override
    public Headline getHeadline() {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build());
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
    }


@@ -104,7 +105,6 @@ public class Document implements GenericSemanticNode {
        return streamAllSubNodes().collect(Collectors.groupingBy(SemanticNode::getType, Collectors.counting()));
    }

-
    @Override
    public String toString() {

@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Footer implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -67,14 +62,4 @@ public class Footer implements GenericSemanticNode {
        return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Header implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public boolean isLeaf() {
@@ -67,14 +62,4 @@ public class Header implements GenericSemanticNode {
        return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Headline implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -74,14 +69,4 @@ public class Headline implements GenericSemanticNode {
        return this;
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,9 +32,6 @@ public class Paragraph implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -65,14 +60,4 @@ public class Paragraph implements GenericSemanticNode {
        return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -37,9 +35,6 @@ public class Section implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -79,14 +74,4 @@ public class Section implements GenericSemanticNode {
                .orElseGet(() -> getParent().getHeadline());
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -398,11 +398,12 @@ public interface SemanticNode {
     */
    default Map<Page, Rectangle2D> getBBox() {

+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        if (isLeaf()) {
-            return getBBoxFromLeafTextBlock();
+            return getBBoxFromLeafTextBlock(bBoxPerPage);
        }

-        return getBBoxFromChildren();
+        return getBBoxFromChildren(bBoxPerPage);
    }


@@ -425,31 +426,25 @@ public interface SemanticNode {

    /**
     * TODO: this produces unwanted results for sections spanning multiple columns.
-     * Computes the Union of the bounding boxes of all children recursively.
+     *
+     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of the BoundingBoxes of all children
     */
-    private Map<Page, Rectangle2D> getBBoxFromChildren() {
+    private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
-        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
-        for (Page page : pages) {
-            Rectangle2D bBoxOnPage = childrenBBoxes.stream()
-                    .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
-                    .map(childBboxPerPage -> childBboxPerPage.get(page))
-                    .collect(RectangleTransformations.collectBBox());
-            bBoxPerPage.put(page, bBoxOnPage);
-        }
-        return bBoxPerPage;
+        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
+            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
+            return map2;
+        }).orElse(bBoxPerPage);
    }


    /**
+     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of all BoundingBoxes of the TextBlock of this node
     */
-    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
+    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
        return bBoxPerPage;
@@ -2,12 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

 import static java.lang.String.format;

-import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -36,14 +34,13 @@ public class Table implements SemanticNode {

    int numberOfRows;
    int numberOfCols;
+
    TextBlock textBlock;

    @Builder.Default
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;

    /**
     * Streams all entities in this table, that appear in a row, which contains any of the provided strings.
@@ -211,6 +208,7 @@ public class Table implements SemanticNode {
        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
    }

+
    /**
     * Streams all TableCells row-wise and filters them with header == true.
     *
@@ -315,12 +313,5 @@ public class Table implements SemanticNode {

        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
    }
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {

-        if (bBoxCache == null) {
-            bBoxCache = SemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
 }
@@ -1,14 +1,12 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -254,8 +252,7 @@ public class TablePageBlock extends AbstractPageBlock {
                if (prevY != null && prevX != null) {
                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));

-                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
-
+                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
                    if (cell.hasMinimumSize()) {
                        row.add(cell);
@@ -276,21 +273,6 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-
-    public boolean intersects(Cell cell1, Cell cell2) {
-        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
-            return false;
-        }
-        double x0 = cell1.getX() + 2;
-        double y0 = cell1.getY() + 2;
-        return (cell2.x + cell2.width > x0 &&
-                cell2.y + cell2.height > y0 &&
-                cell2.x < x0 + cell1.getWidth() -2 &&
-                cell2.y < y0 + cell1.getHeight() -2);
-    }
-
-
-
    @Override
    public String getText() {

@@ -17,6 +17,7 @@ import lombok.SneakyThrows;
@AllArgsConstructor
 public class RedTextPosition {

+    private String textMatrix;
    private float[] position;

    @JsonIgnore
@@ -55,6 +56,8 @@ public class RedTextPosition {

        pos.setFontSizeInPt(textPosition.getFontSizeInPt());

+        pos.setTextMatrix(textPosition.getTextMatrix().toString());
+
        var position = new float[4];

        position[0] = textPosition.getXDirAdj();
@@ -9,6 +9,8 @@ import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;

+import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;

@@ -140,7 +142,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minX value
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMinXDirAdj() {

        return textPositions.get(0).getXDirAdj();
@@ -154,7 +157,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxX value
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMaxXDirAdj() {

        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
@@ -168,7 +172,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMinYDirAdj() {

        return textPositions.get(0).getYDirAdj() - getTextHeight();
@@ -182,7 +187,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMaxYDirAdj() {

        return textPositions.get(0).getYDirAdj();
@@ -190,38 +196,42 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getHeight() {

        return getMaxYDirAdj() - getMinYDirAdj();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getWidth() {

        return getMaxXDirAdj() - getMinXDirAdj();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getFont() {

-        if (textPositions.get(0).getFontName() == null) {
-            return "none";
-        }
        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getFontStyle() {

-        if (textPositions.get(0).getFontName() == null) {
-            return "standard";
-        }
        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);

        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
@@ -233,15 +243,20 @@ public class TextPositionSequence implements CharSequence {
        } else {
            return "standard";
        }
+
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getFontSize() {

        return textPositions.get(0).getFontSizeInPt();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getSpaceWidth() {

        return textPositions.get(0).getWidthOfSpace();
@@ -257,7 +272,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return bounding box of the word in Pdf Coordinate System
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    @SneakyThrows
    public Rectangle getRectangle() {

@@ -8,7 +8,6 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.PageInfo;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;

@@ -25,26 +24,24 @@ public class CvTableParsingAdapter {
        Map<Integer, List<TableCells>> tableCells = new HashMap<>();
        tableServiceResponse.getData()
                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells(), tableData.getPageInfo())));
+                        .addAll(convertTableCells(tableData.getTableCells())));

        return tableCells;
    }


-    private Collection<TableCells> convertTableCells(List<TableCells> tableCells, PageInfo pageInfo) {
+    private Collection<TableCells> convertTableCells(List<TableCells> tableCells) {

        List<TableCells> cvParsedTableCells = new ArrayList<>();

-        tableCells.stream()
-                .filter(cell -> cell.getWidth() < pageInfo.getWidth() * 0.98 && cell.getHeight() < pageInfo.getHeight() * 0.98)
-                .forEach(t -> cvParsedTableCells.add(TableCells.builder()
-                        .y0(t.getY0())
-                        .x1(t.getX1())
-                        .y1(t.getY1())
-                        .x0(t.getX0())
-                        .width(t.getWidth())
-                        .height(t.getHeight())
-                        .build()));
+        tableCells.forEach(t -> cvParsedTableCells.add(TableCells.builder()
+                .y0(t.getY0())
+                .x1(t.getX1())
+                .y1(t.getY1())
+                .x0(t.getX0())
+                .width(t.getWidth())
+                .height(t.getHeight())
+                .build()));

        return cvParsedTableCells;
    }
@@ -12,9 +12,9 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;

 import lombok.RequiredArgsConstructor;
@@ -25,13 +25,10 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RulingCleaningService {

-    private static final float THRESHOLD = 6;
-
-
-    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {
+    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings, float minCharWidth, float maxCharHeight) {

        if (!rulings.isEmpty()) {
-            snapPoints(rulings);
+            snapPoints(rulings, minCharWidth, maxCharHeight);
        }

        List<Ruling> vrs = new ArrayList<>();
@@ -56,11 +53,14 @@ public class RulingCleaningService {
        }
        List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);

-        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
+        return CleanRulings.builder()
+                .vertical(verticalRulingLines)
+                .horizontal(horizontalRulingLines)
+                .build();
    }


-    public void snapPoints(List<? extends Line2D.Float> rulings) {
+    public void snapPoints(List<? extends Line2D.Float> rulings, float xThreshold, float yThreshold) {

        // collect points and keep a Line -> p1,p2 map
        Map<Line2D.Float, Point2D[]> linesToPoints = new HashMap<>();
@@ -81,7 +81,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
+            if (Math.abs(p.getX() - last.get(0).getX()) < xThreshold) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -108,7 +108,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
+            if (Math.abs(p.getY() - last.get(0).getY()) < yThreshold) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -0,0 +1,146 @@
+package com.knecon.fforesight.service.layoutparser.processor.services;
+
+import java.awt.geom.Rectangle2D;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class SectionGridCreatorService {
+
+    public SectionGrid createSectionGrid(Document document) {
+
+        Map<Integer, List<SectionRectangle>> sectionBBox = document.streamAllSubNodesOfType(NodeType.SECTION).map(SemanticNode::getBBox).collect(new SectionGridCollector());
+        Map<Integer, List<SectionRectangle>> paragraphBBox = document.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBBox).collect(new SectionGridCollector());
+        Map<Integer, List<SectionRectangle>> headlineBBox = document.streamAllSubNodesOfType(NodeType.HEADLINE).map(SemanticNode::getBBox).collect(new SectionGridCollector());
+        Map<Integer, List<SectionRectangle>> tableBBox = document.streamAllSubNodesOfType(NodeType.TABLE).map(node -> (Table) node).collect(new TableGridCollector());
+        var sectionGrid = new SectionGrid();
+
+        sectionGrid.setRectanglesPerPage(mergeMapsByConcatenatingLists(//
+                mergeMapsByConcatenatingLists(paragraphBBox, headlineBBox), //
+                mergeMapsByConcatenatingLists(sectionBBox, tableBBox)));
+
+        return sectionGrid;
+    }
+
+
+    private static abstract class GridCollector<T> implements Collector<T, Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> {
+
+        @Override
+        public Supplier<Map<Integer, List<SectionRectangle>>> supplier() {
+
+            return HashMap::new;
+        }
+
+
+        @Override
+        public Function<Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> finisher() {
+
+            return Function.identity();
+        }
+
+
+        @Override
+        public BinaryOperator<Map<Integer, List<SectionRectangle>>> combiner() {
+
+            return SectionGridCreatorService::mergeMapsByConcatenatingLists;
+        }
+
+
+        @Override
+        public Set<Characteristics> characteristics() {
+
+            return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
+        }
+
+    }
+
+    private static class TableGridCollector extends GridCollector<Table> {
+
+        @Override
+        public BiConsumer<Map<Integer, List<SectionRectangle>>, Table> accumulator() {
+
+            return (map, table) -> table.getPages()
+                    .forEach(page -> map.merge(page.getNumber(), List.of(toSectionRectangle(table, page, table.getPages().size())), SectionGridCreatorService::concatLists));
+        }
+
+
+        private static SectionRectangle toSectionRectangle(Table table, Page page, int numberOfParts) {
+
+            Rectangle2D rect = table.getBBox().get(page);
+            List<CellRectangle> tableCellRectangles = table.streamTableCells()
+                    .map(TableCell::getBBox)
+                    .map(map -> map.get(page))
+                    .filter(Objects::nonNull)
+                    .map(rectangle2D -> new CellRectangle(new Point((float) rectangle2D.getX(), (float) rectangle2D.getY()),
+                            (float) rectangle2D.getWidth(),
+                            (float) rectangle2D.getHeight()))
+                    .toList();
+            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()),
+                    (float) rect.getWidth(),
+                    (float) rect.getHeight(),
+                    1,
+                    numberOfParts,
+                    tableCellRectangles);
+        }
+
+    }
+
+    private static class SectionGridCollector extends GridCollector<Map<Page, Rectangle2D>> {
+
+        @Override
+        public BiConsumer<Map<Integer, List<SectionRectangle>>, Map<Page, Rectangle2D>> accumulator() {
+
+            return (mapToKeep, mapToMerge) -> mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page.getNumber(),
+                    List.of(toSectionRectangle(rectangle, mapToMerge.values().size())),
+                    SectionGridCreatorService::concatLists));
+
+        }
+
+
+        private static SectionRectangle toSectionRectangle(Rectangle2D rect, int numberOfParts) {
+
+            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()), (float) rect.getWidth(), (float) rect.getHeight(), 1, numberOfParts, null);
+        }
+
+    }
+
+
+    private static Map<Integer, List<SectionRectangle>> mergeMapsByConcatenatingLists(Map<Integer, List<SectionRectangle>> mapToKeep,
+                                                                                      Map<Integer, List<SectionRectangle>> mapToMerge) {
+
+        mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page, rectangle, SectionGridCreatorService::concatLists));
+        return mapToKeep;
+    }
+
+
+    private static List<SectionRectangle> concatLists(List<SectionRectangle> l1, List<SectionRectangle> l2) {
+
+        return Stream.concat(l1.stream(), l2.stream()).toList();
+    }
+
+}
@@ -12,6 +12,7 @@ import java.util.Set;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -65,17 +66,6 @@ public class TableExtractionService {
    };


-    public boolean contains(Cell cell, double x, double y, double w, double h) {
-
-        if (cell.isEmpty() || w <= 0 || h <= 0) {
-            return false;
-        }
-        double x0 = cell.getX();
-        double y0 = cell.getY();
-        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
-    }
-
-
    /**
     * Finds tables on a page and moves textblocks into cells of the found tables.
     * Note: This algorithm uses Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -89,17 +79,16 @@ public class TableExtractionService {
     * @param cleanRulings The lines used to build the table.
     * @param page         Page object that contains textblocks and statistics.
     */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {
+    public void extractTables(CleanRulings cleanRulings, ClassificationPage page, LayoutParsingType layoutParsingType) {

-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
+        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical(), layoutParsingType);

        List<TextPageBlock> toBeRemoved = new ArrayList<>();

        for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
            TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
            for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && contains(cell,
-                        textBlock.getPdfMinX(),
+                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
                        textBlock.getPdfMinY(),
                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -113,7 +102,7 @@ public class TableExtractionService {
        cells = new ArrayList<>(new HashSet<>(cells));
        DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);

-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);
+        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).toList();

        List<TablePageBlock> tables = new ArrayList<>();
        for (Rectangle area : spreadsheetAreas) {
@@ -146,14 +135,16 @@ public class TableExtractionService {
    }


-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, LayoutParsingType layoutParsingType) {

-        // Fix for 211.pdf
-        for (Ruling r : horizontalRulingLines) {
-            if (r.getX2() < r.getX1()) {
-                double a = r.getX2();
-                r.x2 = (float) r.getX1();
-                r.x1 = (float) a;
+        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
+            // TODO: breaks some tables, for example "1 Abamectin Prr.pdf" try to fix this upstream in RulingCleaningService
+            for (Ruling r : horizontalRulingLines) {
+                if (r.getX2() < r.getX1()) {
+                    double a = r.getX2();
+                    r.x2 = (float) r.getX1();
+                    r.x1 = (float) a;
+                }
            }
        }

@@ -1,8 +1,18 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;

+import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;

-// TODO: figure out, why this fails the build
-// import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -11,12 +21,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;

@Service
@SuppressWarnings("all")
@@ -24,13 +28,12 @@ public class TaasBlockificationService {

    private static final float THRESHOLD = 1f;
    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f; // multiplied with text height
-    private static final float INTERSECTS_Y_THRESHOLD = 4;// 2 * HEIGHT_PADDING // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
+    private static final float INTERSECTS_Y_THRESHOLD =  2 * HEIGHT_PADDING; // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
    private static final int X_GAP_SPLIT_CONSTANT = 50;
    public static final int X_ALIGNMENT_THRESHOLD = 1;
+    public static final int SMALL_Y_GAP_THRESHOLD = 5;
    public static final int NEGATIVE_X_GAP_THRESHOLD = -5;

-    private Pattern listIdentifier = Pattern.compile("^(?:(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)]))|\\uF0B7", Pattern.CASE_INSENSITIVE);
-

    /**
     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
@@ -75,29 +78,16 @@ public class TaasBlockificationService {
        List<TextPageBlock> currentTextBlocksToMerge = new LinkedList<>();
        textBlocksToMerge.add(currentTextBlocksToMerge);
        TextPageBlock previousTextBlock = null;
-        Float lastLineGap = null;
        for (TextPageBlock currentTextBlock : classificationTextBlocks) {
            if (previousTextBlock == null) {
                currentTextBlocksToMerge.add(currentTextBlock);
                previousTextBlock = currentTextBlock;
                continue;
            }
-
-
-            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
-            boolean isListIdentifier = listIdentifierPattern.find();
-
-            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-
-            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
-//            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-
            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
-            boolean alignsXLeft = Math.abs(currentTextBlock.getPdfMinX() - previousTextBlock.getPdfMinX()) < X_ALIGNMENT_THRESHOLD;
-//            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < yGap;
-            if (yGap && sameFont && !isListIdentifier) {
+            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < SMALL_Y_GAP_THRESHOLD;
+            if (alignsXRight && smallYGap) {
                currentTextBlocksToMerge.add(currentTextBlock);
-
            } else {
                currentTextBlocksToMerge = new LinkedList<>();
                currentTextBlocksToMerge.add(currentTextBlock);
@@ -178,8 +168,8 @@ public class TaasBlockificationService {


    private List<TextPageBlock> constructFineGranularTextPageBlocks(List<TextPositionSequence> textPositions,
-                                                                    List<Ruling> horizontalRulingLines,
-                                                                    List<Ruling> verticalRulingLines) {
+                                                                                        List<Ruling> horizontalRulingLines,
+                                                                                        List<Ruling> verticalRulingLines) {

        int indexOnPage = 0;
        List<TextPositionSequence> wordClusterToCombine = new ArrayList<>();
@@ -188,13 +178,13 @@ public class TaasBlockificationService {
        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
        TextPositionSequence prev = null;
        // TODO: make static final constant
-
+        var listIdentitifier = Pattern.compile("\\b(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)])", Pattern.CASE_INSENSITIVE);

        boolean wasSplitted = false;
        Float splitX1 = null;
        for (TextPositionSequence word : textPositions) {

-            Matcher listIdentifierPattern = listIdentifier.matcher(word.toString());
+            Matcher listIdentifierPattern = listIdentitifier.matcher(word.toString());

            boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
            boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());
@@ -18,7 +18,7 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class SearchTextWithTextPositionFactory {

-    public final int HEIGHT_PADDING = 2;
+    public static final int HEIGHT_PADDING = 2;
    // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away.
    // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height.
    // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate
@@ -133,7 +133,7 @@ public class SearchTextWithTextPositionFactory {

    private static void addTextPositionWithFontType(RedTextPosition currentTextPosition, String fontType, List<Integer> fontTypePositions, int stringIdx) {

-        if (currentTextPosition.getFontName() != null && currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
+        if (currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
            fontTypePositions.add(stringIdx);
        }
    }
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -9,7 +8,6 @@ import java.util.Map;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;

@@ -1,34 +1,18 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.parsing;

-import java.awt.color.CMMException;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
+import org.apache.pdfbox.contentstream.operator.color.*;
 import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
 import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
-import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
-import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
-import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
-import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
-import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
+import org.apache.pdfbox.contentstream.operator.state.*;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSNumber;
@@ -37,14 +21,11 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.text.TextPosition;

-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-
-import lombok.Getter;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.color.CMMException;
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;

@Getter
@Slf4j
@@ -55,6 +36,11 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private final List<Ruling> graphicsPath = new ArrayList<>();
    @Setter
    protected PDPage pdpage;
+    private int minCharWidth;
+    private int maxCharWidth;
+    private int minCharHeight;
+    private int maxCharHeight;
+

    private float path_x;
    private float path_y;
@@ -87,6 +73,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        this.addOperator(new SetFontAndSize(this));
        this.addOperator(new SetLineWidth(this));

+
        addOperator(new BeginMarkedContentSequenceWithProperties(this));
 //        addOperator(new BeginMarkedContentSequence(this));
        addOperator(new EndMarkedContentSequence(this));
@@ -245,15 +232,33 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
            }

+            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
+            if (charWidth < minCharWidth) {
+                minCharWidth = charWidth;
+            }
+            if (charWidth > maxCharWidth) {
+                maxCharWidth = charWidth;
+            }
+
+            int charHeight = (int) textPositions.get(i).getHeightDir();
+            if (charHeight < minCharHeight) {
+                minCharHeight = charHeight;
+            }
+            if (charWidth > maxCharHeight) {
+                maxCharHeight = charHeight;
+            }
+
            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
                startIndex++;
                continue;
            }

            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
+            if (i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj()) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                        .getUnicode()
+                        .equals("\t")))) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -261,7 +266,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {

            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                        .getUnicode()
+                        .equals("\t")))) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -271,10 +278,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                    .getUnicode()
                    .equals("\t")) && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                        .getUnicode()
+                        .equals("\t")))) {

                    // Remove false sequence ends (whitespaces)
-                    if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
+                    if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
+                            .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
                        for (TextPosition t : sublist) {
                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                        }
@@ -309,29 +319,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    }


-    public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {
-
-        return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
-    }
-
-
-    public boolean checkIfSequenceContainsOnlyWhitespaces(List<TextPosition> sublist) {
-
-        return !(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                .getUnicode()
-                .equals("\t")));
-    }
-
-
-    public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {
-
-        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
-    }
-
    @Override
    public String getText(PDDocument doc) throws IOException {

+        minCharWidth = Integer.MAX_VALUE;
+        maxCharWidth = 0;
+        minCharHeight = Integer.MAX_VALUE;
+        maxCharHeight = 0;
        textPositionSequences.clear();
        rulings.clear();
        graphicsPath.clear();
@@ -1711,7 +1711,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
        int numberOfStrings = line.size();
        for (int i = 0; i < numberOfStrings; i++) {
            WordWithTextPositions word = line.get(i);
-            word.getTextPositions().sort(Comparator.comparing(TextPosition::getXDirAdj));
+            word.getTextPositions().sort(Comparator.comparing(TextPosition::getX));
            writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
            if (i < numberOfStrings - 1) {
                writeWordSeparator();
@@ -2,13 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor.services.visualizat

 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
+import java.io.OutputStream;
+import java.util.HashSet;
+import java.util.Set;

-import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.cos.COSDictionary;
 import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
@@ -32,9 +31,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.visualization.
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;

-import io.micrometer.observation.Observation;
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -44,31 +40,29 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class ViewerDocumentService {

+
    private static final String LAYER_NAME = "Layout grid";
    private static final int FONT_SIZE = 10;
    public static final float LINE_WIDTH = 1f;

    private final LayoutGridService layoutGridService;
-    private final ObservationRegistry observationRegistry;


    @SneakyThrows
-    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
-    public void createViewerDocument(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {
+    public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream, boolean layerVisibilityDefaultValue) {

-        Path tmpFile = Files.createTempFile("tmpViewerDocument", ".pdf");
-        PDDocument pdDocument = openPDDocument(originFile);
        LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
-
-        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
+        // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
+        // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
+        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
+        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
        PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);

        for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
-
            PDPage pdPage = pdDocument.getPage(pageNumber);
-//
+
            AffineTransform textDeRotationMatrix = getTextDeRotationTransform(pdPage);
-            addLayerToPageResources(pdPage);
+            addLayerToPageRessources(pdPage);

            // We need to save the graphics state before, such that our appended content cannot be affected by previous content streams with side effects,
            // e.g. not escaped matrix transformations.
@@ -121,48 +115,16 @@ public class ViewerDocumentService {
                contentStream.restoreGraphicsState();
                contentStream.endMarkedContent();
            }
-
-            if (pageNumber % 500 == 0 && pageNumber != 0) { // re-open document every once in a while to save on RAM
-                log.info("Incremental save after {} pages", pageNumber);
-                observedIncrementalSave(pdDocument, destinationFile);
-                pdDocument.close();
-                Files.copy(destinationFile.toPath(), tmpFile, StandardCopyOption.REPLACE_EXISTING);
-                pdDocument = openPDDocument(tmpFile.toFile());
-                layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
-            }
-
+            dictionariesToUpdate.add(pdPage.getCOSObject());
+            dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
        }
-        observedIncrementalSave(pdDocument, destinationFile);
-
-        tmpFile.toFile().delete();
-        pdDocument.close();
+        dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
+//        dictionariesToUpdate.add(pdDocument.getDocument().getTrailer());
+        pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
    }


-    private static PDDocument openPDDocument(File tmpFile) throws IOException {
-
-        PDDocument pdDocument;
-        pdDocument = Loader.loadPDF(tmpFile);
-        pdDocument.setAllSecurityToBeRemoved(true);
-        return pdDocument;
-    }
-
-
-    @SneakyThrows
-    private void observedIncrementalSave(PDDocument pdDocument, File outputFile) {
-
-        Observation.createNotStarted("ViewerDocumentService", observationRegistry).contextualName("incremental-save").observe(() -> {
-            try {
-                pdDocument.save(outputFile);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        });
-
-    }
-
-
-    private static void addLayerToPageResources(PDPage pdPage) {
+    private static void addLayerToPageRessources(PDPage pdPage) {

        PDResources resources = pdPage.getResources();
        if (resources == null) {
@@ -183,7 +145,7 @@ public class ViewerDocumentService {
    }


-    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, boolean layerVisibilityDefaultValue) {
+    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate, boolean layerVisibilityDefaultValue) {

        PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
        PDOptionalContentProperties ocprops = catalog.getOCProperties();
@@ -199,7 +161,7 @@ public class ViewerDocumentService {
            ocprops.addGroup(layer);
        }
        ocprops.setGroupEnabled(layer, layerVisibilityDefaultValue);
-//        dictionariesToUpdate.add(catalog.getCOSObject());
+        dictionariesToUpdate.add(catalog.getCOSObject());
        return layer;
    }

@@ -9,7 +9,6 @@ import org.apache.pdfbox.text.TextPosition;

 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -23,7 +22,7 @@ public class MarkedContentUtils {
    public List<Rectangle2D> getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {

        if (markedContents == null) {
-            return Collections.emptyList();
+            return null;
        }

        var markedContentByYPosition = markedContents.stream()
@@ -38,7 +37,7 @@ public class MarkedContentUtils {
                .collect(Collectors.groupingBy(TextPosition::getY));

        if (markedContentByYPosition.isEmpty()) {
-            return Collections.emptyList();
+            return null;
        }

        return markedContentByYPosition.values().stream()
@@ -19,10 +19,9 @@ public final class PositionUtils {

        double threshold = textBlock.getMostPopularWordHeight() * 3;

-        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX()
-                && textBlock.getPdfMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth()
-                && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY()
-                && textBlock.getPdfMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
+        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX() && textBlock.getPdfMaxX() - threshold < btf.getTopLeft()
+                .getX() + btf.getWidth() && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY() && textBlock.getPdfMaxY() - threshold < btf.getTopLeft()
+                .getY() + btf.getHeight()) {
            return true;
        } else {
            return false;
@@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
    application
    id("com.knecon.fforesight.java-conventions")
-    id("org.springframework.boot") version "3.1.5"
+    id("org.springframework.boot") version "3.1.3"
    id("io.spring.dependency-management") version "1.1.3"
    id("org.sonarqube") version "4.3.0.3225"
    id("io.freefair.lombok") version "8.2.2"
@@ -12,43 +12,31 @@ plugins {

 description = "layoutparser-service-server"

-configurations {
-    all {
-        exclude(group = "commons-logging", module = "commons-logging")
-        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
-        exclude(group = "com.iqser.red.commons", module = "logging-commons")
-    }
-}
-
-
 val jacksonVersion = "2.15.2"
 val pdfBoxVersion = "3.0.0"
-val springBootStarterVersion = "3.1.5"

 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
-    implementation("com.knecon.fforesight:tracing-commons:0.3.0")
-    implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
+    implementation("com.iqser.red.commons:storage-commons:2.40.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
    implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
-    implementation("net.logstash.logback:logstash-logback-encoder:7.4")

    // for integration testing only
    testImplementation(project(":layoutparser-service-internal-api"))

-    testImplementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
-    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0")
    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.3")
 }

 /*
@@ -5,7 +5,6 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
@@ -14,11 +13,8 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingService
 import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.aop.ObservedAspect;
-
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
-@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
+@Import({MetricsConfiguration.class, StorageAutoConfiguration.class,  LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

@@ -27,11 +23,4 @@ public class Application {
        SpringApplication.run(Application.class, args);
    }

-
-    @Bean
-    public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
-
-        return new ObservedAspect(observationRegistry);
-    }
-
 }
@@ -24,7 +24,8 @@ public class MessagingConfiguration {
    @Bean
    public Queue layoutparsingResponseQueue() {

-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
    }


@@ -5,19 +5,10 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/

 fforesight.tenants.remote: true

-logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
-
-logging.type: ${LOGGING_TYPE:CONSOLE}
-kubernetes.namespace: ${NAMESPACE:default}
-project.version: 1.0-SNAPSHOT
-
-
 server:
  port: 8080

 spring:
-  application:
-    name: layoutparser-service
  main:
    allow-circular-references: true # FIXME
  rabbitmq:
@@ -41,13 +32,6 @@ management:
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
  endpoints.web.exposure.include: prometheus, health
-  tracing:
-    enabled: ${TRACING_ENABLED:false}
-    sampling:
-      probability: ${TRACING_PROBABILITY:1.0}
-  otlp:
-    tracing:
-      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}


 storage:
@@ -1,17 +0,0 @@
-<configuration>
-
-    <springProperty scope="configuration" name="logType" source="logging.type"/>
-    <springProperty scope="context" name="application.name" source="spring.application.name"/>
-    <springProperty scope="context" name="version" source="project.version"/>
-    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
-    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
-
-    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
-    </appender>
-
-    <root level="INFO">
-        <appender-ref ref="${logType}"/>
-    </root>
-
-</configuration>
@@ -46,11 +46,12 @@ public class BdrJsonBuildTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(File file) {

-        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
-                file,
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                file.toString()));
+        try (PDDocument pdDocument = Loader.loadPDF(file)) {
+            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                    pdDocument,
+                    new ImageServiceResponse(),
+                    new TableServiceResponse()));
+        }
    }


@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

+import org.apache.pdfbox.Loader;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -95,10 +96,9 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                pdfFileResource.getFile(),
+                Loader.loadPDF(pdfFileResource.getFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filePath));
+                new TableServiceResponse()));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -25,8 +25,8 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
    @SneakyThrows
    public void testLayoutParserEndToEnd() {

-        prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        prepareStorage("files/bdr/btd_19_053_1905391.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.TAAS);
        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
    }
@@ -6,7 +6,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;

-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -18,9 +17,7 @@ import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest
 import lombok.SneakyThrows;

 public class DocumentDataTests extends BuildDocumentTest {
-
    @Test
-    @Disabled // This test takes waaaaaay too long, it's ridiculous
    @SneakyThrows
    public void createDocumentDataForAllFiles() {

@@ -39,12 +36,11 @@ public class DocumentDataTests extends BuildDocumentTest {
        for (String pdfFileName : pdfFileNames) {
            System.out.println(pdfFileName);
            DocumentData documentData = DocumentDataMapper.toDocumentData(buildGraph(resource.getFile().toPath().getParent().relativize(Path.of(pdfFileName)).toString()));
-            File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
+                        File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
            outputFile.toPath().getParent().toFile().mkdirs();
            try (var out = new FileOutputStream(outputFile.toString().replace(".pdf", ".json"))) {
                ObjectMapperFactory.create().writeValue(out, documentData);
            }
        }
    }
-
 }
@@ -5,6 +5,7 @@ import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;

+import org.apache.pdfbox.Loader;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;
@@ -55,10 +56,9 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
    private void writeJsons(Path filename) {

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
+                Loader.loadPDF(filename.toFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
+                new TableServiceResponse()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -47,7 +47,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentTest {
    @Disabled
    public void visualizeCraftedDocument() {

-        String filename = "files/1 Abamectin_prr.pdf";
+        String filename = "files/crafted document.pdf";
        visualizePdf(filename);
    }

@@ -53,7 +53,7 @@ public class ExtractMarkedContentTest {
    @SneakyThrows
    public void testExtractTestWPhromma() throws IOException {
        System.out.printf("\n\n===\n%s\n===\n", "testWPhromma.pdf");
-        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());

            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();

@@ -70,8 +70,6 @@ public class ExtractMarkedContentTest {

            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
            showStructure(root, markedContents);
-            document.close();
-        }
    }

    /**
@@ -89,8 +87,7 @@ public class ExtractMarkedContentTest {
    @Test
    public void testExtractResMultipage() throws IOException {
        System.out.printf("\n\n===\n%s\n===\n", "res_multipage.pdf");
-
-        try(PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());

            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();

@@ -107,8 +104,6 @@ public class ExtractMarkedContentTest {

            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
            showStructure(root, markedContents);
-            document.close();
-        }
    }

    /**
@@ -126,7 +121,7 @@ public class ExtractMarkedContentTest {
    @Test
    public void testExtractDailyReport() throws IOException {
        System.out.printf("\n\n===\n%s\n===\n", "Daily Report.pdf");
-        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());

            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();

@@ -143,9 +138,7 @@ public class ExtractMarkedContentTest {

            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
            showStructure(root, markedContents);
-            document.close();
        }
-    }

    /**
     * @see #testExtractTestWPhromma()
@@ -1,8 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

-import java.io.File;
+import java.io.FileOutputStream;
 import java.nio.file.Path;

+import org.apache.pdfbox.Loader;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -17,16 +19,18 @@ import lombok.SneakyThrows;
 public class ViewerDocumentTest extends BuildDocumentTest {

    @Test
+    @Disabled
    @SneakyThrows
    public void testViewerDocument() {

-        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
-        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
        LayoutGridService layoutGridService = new LayoutGridService();
-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
-        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
-        var documentFile = new ClassPathResource(fileName).getFile();
-        viewerDocumentService.createViewerDocument(documentFile, document, new File(tmpFileName), true);
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
+        String fileName = "files/bdr/notMergedParagraphs.pdf";
+        Document document = buildGraph(fileName, LayoutParsingType.TAAS);
+        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
+        try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getFile()); var out = new FileOutputStream(tmpFileName)) {
+            viewerDocumentService.createViewerDocument(pdDocument, document, out, true);
+        }
    }

 }
@@ -27,14 +27,14 @@ public class TextPositionSequenceTest {
            + "  \"pageWidth\": 600\n"  //
            + "}";

-    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper());
+    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper(), new StorageProperties());


    @Test
    @SneakyThrows
    public void testDeserializationWithJackson() {

-        TextPositionSequence textPositionSequence = objectSerializer.deserialize(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithJackson(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
                TextPositionSequence.class);

        assertPropertiesAfterJsonDeserialization(textPositionSequence);
@@ -1,26 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.server.segmentation;

-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.awt.geom.Rectangle2D;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -36,8 +15,19 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
-
 import lombok.SneakyThrows;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import java.awt.geom.Rectangle2D;
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;

 public class PdfSegmentationServiceTest extends AbstractTest {

@@ -60,14 +50,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    private SectionsBuilderService sectionsBuilderService;


-    @SneakyThrows
-    public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {

-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
                originDocument,
                new ImageServiceResponse(),
-                tableServiceResponse,
-                "document");
+                new TableServiceResponse());

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -77,58 +65,6 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


-    @SneakyThrows
-    public ClassificationDocument buildClassificationDocument(File originDocument) {
-
-        return buildClassificationDocument(originDocument, new TableServiceResponse());
-    }
-
-
-    @Test
-    public void tablesToHtmlDebugger() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");
-
-    }
-
-
-    @Test
-    public void tablesToHtmlDebuggerWithCVResponse() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
-        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
-        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
-
-        toHtml(document, "/tmp/ScanRotationBorder.html");
-    }
-
-
-    @Test
-    public void testScanRotationBorderIsIgnored() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
-        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
-        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
-        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
-
-        // Quality of the table parsing is not good, because the file is rotated at scanning.
-        // We only asset that the table border is not the page border.
-        tables.forEach(table -> {
-            assertThat(table.getMinX()).isGreaterThan(75);
-            assertThat(table.getMaxX()).isLessThan(512);
-        });
-    }
-
-
    @Test
    @SneakyThrows
    public void testMapping() {
@@ -156,7 +92,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(table.getColCount()).isEqualTo(6);
@@ -170,7 +106,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -188,7 +124,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(9);
@@ -206,7 +142,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -219,36 +155,19 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


-    @Test // Non-sense test
+    @Test
    public void testDoc56Page170() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

        validateTable(document, 0, 1, 1, 0, 0);
        validateTable(document, 1, 2, 2, 0, 0);
-        validateTable(document, 2, 6, 20, 0, 0);
-        validateTable(document, 3, 7, 31, 0, 0);
-
-    }
-
-
-    @Test
-    public void testDoc211() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/211.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 4);
-
-        validateTable(document, 0, 5, 4, 0, 0);
-        validateTable(document, 1, 5, 15, 14, 0);
-        validateTable(document, 2, 5, 14, 11, 0);
-        validateTable(document, 3, 5, 3, 0, 0);
+        validateTable(document, 2, 7, 20, 0, 140);
+        validateTable(document, 3, 8, 31, 0, 170);

    }

@@ -258,11 +177,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

-        validateTable(document, 0, 8, 8, 0, 0);
+        validateTable(document, 0, 8, 8, 0, 2);

        List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
                        "Author, date",
@@ -272,18 +191,18 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                        "Method meets analytical validation criteria",
                        "Remarks (in case validation criteria are not met)",
                        "Acceptability of the method"),
-                Arrays.asList("Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                Arrays.asList("",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
                        "Evans P.G. 2001 TMJ4569B, VV-323245",
                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
-                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
                        "Y",
                        "N/A",
@@ -299,9 +218,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        toHtml(document, "/tmp/html.html");
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

@@ -314,29 +231,17 @@ public class PdfSegmentationServiceTest extends AbstractTest {


    @Test
-    @Disabled // FIXME Fake Redactions leads to more cells, no solution for this currently
    public void testDocA20622APartB9Page185() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

-        validateTableSize(document, 1);
+        validateTableSize(document, 2);

-        validateTable(document, 0, 7, 4, 0, 0);
-    }
+        validateTable(document, 0, 5, 5, 0, 23);
+        validateTable(document, 1, 11, 9, 0, 36);

-
-    @Test
-    public void testDocA20622APartB9Page185FixedDoc() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185_fixed.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-
-        validateTableSize(document, 1);
-
-        validateTable(document, 0, 7, 4, 0, 0);
    }


@@ -345,7 +250,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izZRMS (CZ) fRR Part B7_Page123.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 6);

@@ -364,7 +269,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04_Page11.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 3);

@@ -380,7 +285,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10_Page532.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 9, 0, 0);
@@ -393,7 +298,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -407,7 +312,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 6, 7, 0);
@@ -420,10 +325,10 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
-        validateTable(document, 0, 10, 6, 0, 0);
+        validateTable(document, 0, 10, 6, 0, 1);

    }

@@ -433,7 +338,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);
        validateTable(document, 0, 2, 2, 0, 0);
@@ -448,7 +353,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -464,7 +369,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -479,7 +384,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -494,7 +399,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -509,7 +414,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

@@ -526,7 +431,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -541,12 +446,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

-        validateTable(document, 0, 6, 8, 0, 0);
-        validateTable(document, 1, 6, 8, 0, 0);
+        validateTable(document, 0, 6, 8, 0, 2);
+        validateTable(document, 1, 6, 8, 0, 1);

    }

@@ -556,7 +461,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -570,7 +475,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");

-        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -579,37 +484,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


-    @SneakyThrows
-    private void toHtml(ClassificationDocument document, String filename) {
-
-        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
-        StringBuilder sb = new StringBuilder();
-
-        int currentPage = 1;
-        for (var table : tables) {
-            if (currentPage != table.getPage()) {
-                currentPage = table.getPage();
-                sb.append("---------------------- Page ").append(currentPage).append("--------------\n");
-            }
-            sb.append("\n\n");
-            sb.append(table.getTextAsHtml());
-        }
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(Path.of(filename).toFile())) {
-            fileOutputStream.write(sb.toString().getBytes());
-        }
-    }
-
-
    private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {

        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
        List<List<Cell>> rows = table.getRows();
        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();

-        for (List<Cell> row : table.getRows()) {
-            row.forEach(r -> System.out.println(r.toString()));
-        }
        assertThat(emptyCellsFoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);

        assertThat(table.getColCount()).isEqualTo(colCount);
@@ -1,39 +1,21 @@
 package com.knecon.fforesight.service.layoutparser.server.services;

-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;

-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Test;
-import org.springframework.core.io.ClassPathResource;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
-import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
-import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;

-public class RulingCleaningServiceTest extends BuildDocumentTest {
+public class RulingCleaningServiceTest {

    @Test
 //    @Disabled
@@ -43,98 +25,13 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
        String fileName = "files/211.pdf";
        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
-        RulingCleaningService rulingCleaningService = new RulingCleaningService();
        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
+
+        RulingCleaningService rulingCleaningService = new RulingCleaningService();
        List<CleanRulings> cleanRulingsPerPage = new LinkedList<>();
        for (PageContents pageContent : pageContents) {
-            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings()));
+            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings(), 8, 20));
        }
-
-    }
-
-
-    @Test
-    @SneakyThrows
-    public void testTableExtraction() {
-
-        LayoutGridService layoutGridService = new LayoutGridService();
-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
-
-        ClassPathResource resource = new ClassPathResource("files");
-        List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
-                .filter(path -> path.getFileName().toString().endsWith(".pdf"))
-                .map(Path::toAbsolutePath)
-                .map(Path::toString)
-                .toList();
-
-        for (int i = 0; i < pdfFileNames.size(); i++) {
-            writeJsons(Path.of(pdfFileNames.get(i)));
-        }
-    }
-
-
-    @SneakyThrows
-    private void writeJsons(Path filename) {
-
-        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
-        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
-        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
-        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
-        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
-            String tmpFileNameBefore = "C:/Users/YANNIK~1/AppData/Local/Temp/before." + filename.getFileName().toString();
-            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
-                PdfDraw.drawDocumentGraph(pdDocument, documentGraphBefore);
-                pdDocument.save(tmpFileNameBefore);
-            }
-            String tmpFileNameAfter = "C:/Users/YANNIK~1/AppData/Local/Temp/after." + filename.getFileName().toString();
-            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
-                PdfDraw.drawDocumentGraph(pdDocument, documentGraphAfter);
-                pdDocument.save(tmpFileNameAfter);
-
-            }
-        }
-    }
-
-
-    @SneakyThrows
-    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2, String pdfName) {
-
-        List listStructure1 = structure1.streamAllEntries()
-                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
-                .map(DocumentStructure.EntryData::getProperties)
-                .map(properties -> {
-                    var builder = Table.builder();
-                    PropertiesMapper.parseTableProperties(properties, builder);
-                    return builder.build();
-                })
-                .toList();
-
-        List listStructure2 = structure2.streamAllEntries()
-                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
-                .map(DocumentStructure.EntryData::getProperties)
-                .map(properties -> {
-                    var builder = Table.builder();
-                    PropertiesMapper.parseTableProperties(properties, builder);
-                    return builder.build();
-                })
-                .toList();
-
-        for (int i = 0; i < listStructure1.size(); i++) {
-            Table tableNode1 = (Table) listStructure1.get(i);
-            Table tableNode2 = (Table) listStructure2.get(i);
-            if (tableNode1.getNumberOfRows() != tableNode2.getNumberOfRows() || tableNode1.getNumberOfCols() != tableNode2.getNumberOfCols()) {
-                return false;
-            }
-        }
-        return true;
    }

 }
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

 import java.io.InputStream;
-import java.util.Map;
 import java.util.Optional;

 import org.junit.jupiter.api.AfterEach;
@@ -13,17 +12,13 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;

-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
@@ -58,6 +53,7 @@ public abstract class AbstractTest {
    protected final static String POSITION_FILE_ID = "positions";
    protected final static String PAGES_FILE_ID = "pages";
    protected final static String TENANT_ID = "tenant";
+    protected final static String SECTION_GRID_ID = "section";
    protected final static String VIEWER_DOCUMENT_ID = "viewer";
    protected final static String SIMPLIFIED_ID = "simplified";

@@ -74,6 +70,7 @@ public abstract class AbstractTest {
                .positionBlockFileStorageId(POSITION_FILE_ID)
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }
@@ -107,11 +104,9 @@ public abstract class AbstractTest {
        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
    }

-
    protected LayoutParsingRequest buildDefaultLayoutParsingRequest(LayoutParsingType layoutParsingType) {

        return LayoutParsingRequest.builder()
-                .identifier(Map.of("fileId", "1337"))
                .layoutParsingType(layoutParsingType)
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
@@ -121,11 +116,11 @@ public abstract class AbstractTest {
                .positionBlockFileStorageId(POSITION_FILE_ID)
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }

-
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile) {

@@ -162,13 +157,6 @@ public abstract class AbstractTest {
    @ComponentScan("com.knecon.fforesight.service.layoutparser")
    public static class TestConfiguration {

-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
    }

 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

-import java.io.File;
+import java.io.InputStream;

+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

@@ -23,9 +25,11 @@ public abstract class BuildDocumentTest extends AbstractTest {
    @SneakyThrows
    protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {

-        File fileResource = new ClassPathResource(filename).getFile();
+        ClassPathResource fileResource = new ClassPathResource(filename);
        prepareStorage(filename);
-        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), filename);
+        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream.readAllBytes())) {
+            return layoutParsingPipeline.parseLayout(layoutParsingType, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse());
+        }
    }


@@ -0,0 +1,124 @@
+package com.knecon.fforesight.service.layoutparser.server.utils;
+
+import static java.io.File.createTempFile;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@Service
+public class FileSystemBackedStorageService implements StorageService {
+
+    private final Map<String, File> dataMap = new HashMap<>();
+
+
+    public FileSystemBackedStorageService() {
+
+    }
+
+
+    @SneakyThrows
+    @Override
+    public void downloadTo(String tenantId, String objectId, File destinationFile) {
+
+        var res = dataMap.get(objectId);
+        if (res == null) {
+            throw new StorageObjectDoesNotExist(new RuntimeException());
+        }
+
+        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
+    }
+
+
+    @Override
+    public void deleteObject(String tenantId, String objectId) {
+
+        dataMap.remove(objectId);
+    }
+
+
+    @Override
+    public boolean objectExists(String tenantId, String objectId) {
+
+        return dataMap.containsKey(objectId);
+    }
+
+
+    @Override
+    @SneakyThrows
+    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
+
+        File tempFile = createTempFile("test", ".tmp");
+        getMapper().writeValue(new FileOutputStream(tempFile), any);
+        dataMap.put(objectId, tempFile);
+    }
+
+
+    private ObjectMapper getMapper() {
+
+        return ObjectMapperFactory.create();
+    }
+
+
+    @Override
+    @SneakyThrows
+    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
+
+        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
+            throw new StorageObjectDoesNotExist("Stored object not found");
+        }
+        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
+    }
+
+
+    public List<String> listPaths() {
+
+        return new ArrayList<>(dataMap.keySet());
+    }
+
+
+    public List<String> listFilePaths() {
+
+        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
+    }
+
+
+    @Override
+    @SneakyThrows
+    public void storeObject(String tenantId, String objectId, InputStream stream) {
+
+        File tempFile = createTempFile("test", ".tmp");
+
+        try (var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.copy(stream, fileOutputStream);
+        }
+
+        dataMap.put(objectId, tempFile);
+    }
+
+
+    public void clearStorage() {
+
+        this.dataMap.forEach((k, v) -> {
+            v.delete();
+        });
+        this.dataMap.clear();
+    }
+
+}
@@ -7,9 +7,6 @@ fforesight.tenants.remote: true
 server:
  port: 8080

-logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
-logging.type: ${LOGGING_TYPE:CONSOLE}
-
 spring:
  main:
    allow-circular-references: true # FIXME
@@ -0,0 +1,16 @@
+<Configuration>
+
+    <Appenders>
+        <Console name="CONSOLE" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="warn">
+            <AppenderRef ref="CONSOLE"/>
+        </Root>
+        <Logger name="com.iqser" level="info"/>
+    </Loggers>
+
+</Configuration>
@@ -1,17 +0,0 @@
-<configuration>
-
-    <springProperty scope="configuration" name="logType" source="logging.type"/>
-    <springProperty scope="context" name="application.name" source="spring.application.name"/>
-    <springProperty scope="context" name="version" source="project.version"/>
-    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
-    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
-
-    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
-    </appender>
-
-    <root level="INFO">
-        <appender-ref ref="${logType}"/>
-    </root>
-
-</configuration>
@@ -2,14 +2,7 @@
 dir=${PWD##*/}
 gradle assemble

-# Get the current Git branch
-branch=$(git rev-parse --abbrev-ref HEAD)
+buildNumber=${1:-1}

-# Get the short commit hash (first 5 characters)
-commit_hash=$(git rev-parse --short=5 HEAD)
-
-# Combine branch and commit hash
-buildName="${USER}-${branch}-${commit_hash}"
-
-gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName --no-build-cache
-echo "nexus.knecon.com:5001/ff/${dir}-service-server:$buildName"
+gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$USER-$buildNumber
+echo "nexus.knecon.com:5001/red/${dir}-server-v1:$USER-$buildNumber"