Merge branch 'RED-1137' into 'main'

RED-1137: Do not observe actuator endpoints See merge request fforesight/layout-parser!92
RED-1137: Do not observe actuator endpoints
2023-12-20 14:11:31 +01:00 · 2023-12-20 14:05:00 +01:00 · 2023-12-20 12:40:00 +01:00 · 2023-12-20 12:40:00 +01:00 · 2023-12-13 16:05:01 +01:00 · 2023-12-13 15:31:26 +01:00
46 changed files with 906 additions and 395 deletions
@@ -10,7 +10,7 @@ deploy:
  script:
    - echo "Building with gradle version ${BUILDVERSION}"
    - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
  artifacts:
    reports:
@@ -24,6 +24,8 @@ tasks.named<Test>("test") {
    reports {
        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
    }
+    minHeapSize = "512m"
+    maxHeapSize = "2048m"
 }

 tasks.test {
@@ -15,8 +15,8 @@ dependencies {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
-    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.40.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")

    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -3,13 +3,16 @@ package com.knecon.fforesight.service.layoutparser.processor;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
-import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;

+import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
@@ -51,91 +54,121 @@ import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDF
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;

+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

+@SuppressWarnings("PMD.CloseResource")
@Slf4j
@Service
@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class LayoutParsingPipeline {

-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final CvTableParsingAdapter cvTableParsingAdapter;
-    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final SectionsBuilderService sectionsBuilderService;
-    private final TaasClassificationService taasClassificationService;
-    private final RedactManagerClassificationService redactManagerClassificationService;
-    private final DocuMineClassificationService docuMineClassificationService;
-    private final SimplifiedSectionTextService simplifiedSectionTextService;
-    private final BodyTextFrameService bodyTextFrameService;
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final TaasBlockificationService taasBlockificationService;
-    private final DocuMineBlockificationService docuMineBlockificationService;
-    private final RedactManagerBlockificationService redactManagerBlockificationService;
-    private final ViewerDocumentService viewerDocumentService;
+    ImageServiceResponseAdapter imageServiceResponseAdapter;
+    CvTableParsingAdapter cvTableParsingAdapter;
+    LayoutParsingStorageService layoutParsingStorageService;
+    SectionsBuilderService sectionsBuilderService;
+    TaasClassificationService taasClassificationService;
+    RedactManagerClassificationService redactManagerClassificationService;
+    DocuMineClassificationService docuMineClassificationService;
+    SimplifiedSectionTextService simplifiedSectionTextService;
+    BodyTextFrameService bodyTextFrameService;
+    RulingCleaningService rulingCleaningService;
+    TableExtractionService tableExtractionService;
+    TaasBlockificationService taasBlockificationService;
+    DocuMineBlockificationService docuMineBlockificationService;
+    RedactManagerBlockificationService redactManagerBlockificationService;
+    ViewerDocumentService viewerDocumentService;
+    ObservationRegistry observationRegistry;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {

        long start = System.currentTimeMillis();
+        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

-        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
-            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
-            }
+        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
+        File viewerDocumentFile = File.createTempFile("viewer_document", ".pdf");

-            TableServiceResponse tableServiceResponse = new TableServiceResponse();
-            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
-            }
-
-            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
-            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-
-            int numberOfPages = originDocument.getNumberOfPages();
-
-            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
-
-            try (var out = new ByteArrayOutputStream()) {
-                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out, false);
-                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
-            }
-
-            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
-                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
-                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
-            }
-
-            return LayoutParsingFinishedEvent.builder()
-                    .identifier(layoutParsingRequest.identifier())
-                    .numberOfPages(numberOfPages)
-                    .duration(System.currentTimeMillis() - start)
-                    .message(format("""
-                                    Layout parsing has finished in %.02f s.
-                                    identifiers: %s
-                                    %s
-                                    Files have been saved with Ids:
-                                    Structure: %s
-                                    Text: %s
-                                    Positions: %s
-                                    PageData: %s
-                                    Simplified Text: %s
-                                    Viewer Doc: %s""",
-                            ((float) (System.currentTimeMillis() - start)) / 1000,
-                            layoutParsingRequest.identifier(),
-                            buildSemanticNodeCountMessage(numberOfPages, documentGraph.buildSemanticNodeCounts()),
-                            layoutParsingRequest.structureFileStorageId(),
-                            layoutParsingRequest.textBlockFileStorageId(),
-                            layoutParsingRequest.positionBlockFileStorageId(),
-                            layoutParsingRequest.pageFileStorageId(),
-                            layoutParsingRequest.simplifiedTextStorageId(),
-                            layoutParsingRequest.viewerDocumentStorageId()))
-                    .build();
+        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
        }
+
+        TableServiceResponse tableServiceResponse = new TableServiceResponse();
+        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+        }
+
+        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+                originFile,
+                imageServiceResponse,
+                tableServiceResponse,
+                layoutParsingRequest.identifier().toString());
+        log.info("Building document graph for {}", layoutParsingRequest.identifier());
+
+        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
+
+        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
+        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
+        viewerDocumentService.createViewerDocument(originFile, documentGraph, viewerDocumentFile, false);
+        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+            log.info("Building research document data for {}", layoutParsingRequest.identifier());
+            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+        }
+
+        originFile.delete();
+        viewerDocumentFile.delete();
+
+        return LayoutParsingFinishedEvent.builder()
+                .identifier(layoutParsingRequest.identifier())
+                .numberOfPages(documentGraph.getNumberOfPages())
+                .duration(System.currentTimeMillis() - start)
+                .message(format("""
+                                Layout parsing has finished in %.02f s.
+                                identifiers: %s
+                                %s
+                                Files have been saved with Ids:
+                                Structure: %s
+                                Text: %s
+                                Positions: %s
+                                PageData: %s
+                                Simplified Text: %s
+                                Viewer Doc: %s""",
+                        ((float) (System.currentTimeMillis() - start)) / 1000,
+                        layoutParsingRequest.identifier(),
+                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
+                        layoutParsingRequest.structureFileStorageId(),
+                        layoutParsingRequest.textBlockFileStorageId(),
+                        layoutParsingRequest.positionBlockFileStorageId(),
+                        layoutParsingRequest.pageFileStorageId(),
+                        layoutParsingRequest.simplifiedTextStorageId(),
+                        layoutParsingRequest.viewerDocumentStorageId()))
+                .build();
+
+    }
+
+
+    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
+
+        AtomicReference<Document> documentReference = new AtomicReference<>();
+
+        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
+            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
+        });
+
+        return documentReference.get();
    }


@@ -143,32 +176,47 @@ public class LayoutParsingPipeline {

        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
                numberOfPages,
-                semanticNodeCounts.get(NodeType.SECTION),
-                semanticNodeCounts.get(NodeType.HEADLINE),
-                semanticNodeCounts.get(NodeType.PARAGRAPH),
-                semanticNodeCounts.get(NodeType.TABLE),
-                semanticNodeCounts.get(NodeType.TABLE_CELL),
-                semanticNodeCounts.get(NodeType.HEADER),
-                semanticNodeCounts.get(NodeType.FOOTER));
+                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
    }


    @SneakyThrows
+    @Observed(name = "LayoutParsingPipeline", contextualName = "parse-layout")
    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
-                                              PDDocument originDocument,
+                                              File originFile,
                                              ImageServiceResponse imageServiceResponse,
-                                              TableServiceResponse tableServiceResponse) {
+                                              TableServiceResponse tableServiceResponse,
+                                              String identifier) {

+        PDDocument originDocument = openDocument(originFile);
+        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);

        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

-        originDocument.setAllSecurityToBeRemoved(true);
        long pageCount = originDocument.getNumberOfPages();

        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+
+            if (pageNumber % 100 == 0) {
+                // re-open document every once in a while to save on RAM. This has no significant performance impact.
+                // This is due to PDFBox caching all images and some other stuff with Soft References. This dereferences them and forces the freeing of memory.
+                originDocument.close();
+                originDocument = openDocument(originFile);
+            }
+
+            if (pageNumber % 100 == 0 || pageNumber == pageCount || pageNumber == 1) {
+                log.info("Extracting text on Page {} for {}", pageNumber, identifier);
+            }
+
            classificationDocument.setPages(classificationPages);
            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
            PDPage pdPage = originDocument.getPage(pageNumber - 1);
@@ -187,10 +235,7 @@ public class LayoutParsingPipeline {
            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);

            PDRectangle cropbox = pdPage.getCropBox();
-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                    stripper.getRulings(),
-                    stripper.getMinCharWidth(),
-                    stripper.getMaxCharHeight());
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());

            ClassificationPage classificationPage = switch (layoutParsingType) {
                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
@@ -213,28 +258,50 @@ public class LayoutParsingPipeline {
                imageServiceResponseAdapter.findOcr(classificationPage);
            }

-            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            tableExtractionService.extractTables(cleanRulings, classificationPage);
+
            buildPageStatistics(classificationPage);
            increaseDocumentStatistics(classificationPage, classificationDocument);

            classificationPages.add(classificationPage);
        }

-        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+        originDocument.close();

+        log.info("Calculating BodyTextFrame for {}", identifier);
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+        log.info("Classify TextBlocks for {}", identifier);
        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }

+        log.info("Building Sections for {}", identifier);
        sectionsBuilderService.buildSections(classificationDocument);
        sectionsBuilderService.addImagesToSections(classificationDocument);
-
        return classificationDocument;
    }


+    private void addNumberOfPagesToTrace(int numberOfPages, long size) {
+
+        if (observationRegistry.getCurrentObservation() != null) {
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileSize", String.valueOf(size));
+        }
+    }
+
+
+    @SneakyThrows
+    private PDDocument openDocument(File originFile) {
+
+        PDDocument document = Loader.loadPDF(originFile);
+        document.setAllSecurityToBeRemoved(true);
+        return document;
+    }
+
+
    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {

        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
@@ -246,9 +313,9 @@ public class LayoutParsingPipeline {

    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {

-//        if (!classificationPage.isLandscape()) {
+        if (!classificationPage.isLandscape()) {
            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-//        }
+        }
        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor;

-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileOutputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -11,7 +9,6 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;

-import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;
@@ -26,6 +23,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -39,16 +37,18 @@ public class LayoutParsingStorageService {
    private final ObjectMapper objectMapper;


-    public PDDocument getOriginFile(String storageId) throws IOException {
+    public PDDocument getOriginDocument(String storageId) throws IOException {

-        try (var originDocumentInputStream = getObject(storageId)) {
-            File tempFile = createTempFile("document", ".pdf");
-            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
-                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
-                originDocumentInputStream.close();
-            }
-            return Loader.loadPDF(tempFile);
-        }
+        return Loader.loadPDF(getOriginFile(storageId));
+    }
+
+
+    @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
+    public File getOriginFile(String storageId) throws IOException {
+
+        File tempFile = createTempFile("document", ".pdf");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return tempFile;
    }


@@ -74,6 +74,7 @@ public class LayoutParsingStorageService {
    }


+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
@@ -83,7 +84,6 @@ public class LayoutParsingStorageService {
    }


-
    public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.researchDocumentStorageId(), researchDocumentData);
@@ -115,6 +115,7 @@ public class LayoutParsingStorageService {
    }


+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-simplified-text")
    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
@@ -132,9 +133,10 @@ public class LayoutParsingStorageService {


    @SneakyThrows
-    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-viewer-document")
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, File out) {

-        try (var in = new ByteArrayInputStream(out.toByteArray())) {
+        try (var in = new FileInputStream(out)) {

            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
        }
@@ -10,7 +10,6 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

-import com.amazonaws.services.kms.model.NotFoundException;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -84,7 +83,7 @@ public class Document implements GenericSemanticNode {
    @Override
    public Headline getHeadline() {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build());
    }


@@ -105,6 +104,7 @@ public class Document implements GenericSemanticNode {
        return streamAllSubNodes().collect(Collectors.groupingBy(SemanticNode::getType, Collectors.counting()));
    }

+
    @Override
    public String toString() {

@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Footer implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -62,4 +67,14 @@ public class Footer implements GenericSemanticNode {
        return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Header implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public boolean isLeaf() {
@@ -62,4 +67,14 @@ public class Header implements GenericSemanticNode {
        return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Headline implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -69,4 +74,14 @@ public class Headline implements GenericSemanticNode {
        return this;
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -32,6 +34,9 @@ public class Paragraph implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -60,4 +65,14 @@ public class Paragraph implements GenericSemanticNode {
        return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -35,6 +37,9 @@ public class Section implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -74,4 +79,14 @@ public class Section implements GenericSemanticNode {
                .orElseGet(() -> getParent().getHeadline());
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -398,12 +398,11 @@ public interface SemanticNode {
     */
    default Map<Page, Rectangle2D> getBBox() {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        if (isLeaf()) {
-            return getBBoxFromLeafTextBlock(bBoxPerPage);
+            return getBBoxFromLeafTextBlock();
        }

-        return getBBoxFromChildren(bBoxPerPage);
+        return getBBoxFromChildren();
    }


@@ -426,25 +425,31 @@ public interface SemanticNode {

    /**
     * TODO: this produces unwanted results for sections spanning multiple columns.
-     *
-     * @param bBoxPerPage initial empty BoundingBox
+     * Computes the Union of the bounding boxes of all children recursively.
     * @return The union of the BoundingBoxes of all children
     */
-    private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {
+    private Map<Page, Rectangle2D> getBBoxFromChildren() {

-        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
-            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
-            return map2;
-        }).orElse(bBoxPerPage);
+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
+        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
+        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
+        for (Page page : pages) {
+            Rectangle2D bBoxOnPage = childrenBBoxes.stream()
+                    .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
+                    .map(childBboxPerPage -> childBboxPerPage.get(page))
+                    .collect(RectangleTransformations.collectBBox());
+            bBoxPerPage.put(page, bBoxOnPage);
+        }
+        return bBoxPerPage;
    }


    /**
-     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of all BoundingBoxes of the TextBlock of this node
     */
-    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
+    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {

+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
        return bBoxPerPage;
@@ -2,10 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

 import static java.lang.String.format;

+import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -34,13 +36,14 @@ public class Table implements SemanticNode {

    int numberOfRows;
    int numberOfCols;
-
    TextBlock textBlock;

    @Builder.Default
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;

    /**
     * Streams all entities in this table, that appear in a row, which contains any of the provided strings.
@@ -208,7 +211,6 @@ public class Table implements SemanticNode {
        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
    }

-
    /**
     * Streams all TableCells row-wise and filters them with header == true.
     *
@@ -313,5 +315,12 @@ public class Table implements SemanticNode {

        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
    }
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {

+        if (bBoxCache == null) {
+            bBoxCache = SemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
 }
@@ -1,12 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -252,7 +254,8 @@ public class TablePageBlock extends AbstractPageBlock {
                if (prevY != null && prevX != null) {
                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));

-                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
+                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
+
                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
                    if (cell.hasMinimumSize()) {
                        row.add(cell);
@@ -273,6 +276,21 @@ public class TablePageBlock extends AbstractPageBlock {
    }


+
+    public boolean intersects(Cell cell1, Cell cell2) {
+        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
+            return false;
+        }
+        double x0 = cell1.getX() + 2;
+        double y0 = cell1.getY() + 2;
+        return (cell2.x + cell2.width > x0 &&
+                cell2.y + cell2.height > y0 &&
+                cell2.x < x0 + cell1.getWidth() -2 &&
+                cell2.y < y0 + cell1.getHeight() -2);
+    }
+
+
+
    @Override
    public String getText() {

@@ -17,7 +17,6 @@ import lombok.SneakyThrows;
@AllArgsConstructor
 public class RedTextPosition {

-    private String textMatrix;
    private float[] position;

    @JsonIgnore
@@ -56,8 +55,6 @@ public class RedTextPosition {

        pos.setFontSizeInPt(textPosition.getFontSizeInPt());

-        pos.setTextMatrix(textPosition.getTextMatrix().toString());
-
        var position = new float[4];

        position[0] = textPosition.getXDirAdj();
@@ -9,8 +9,6 @@ import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;

-import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;

@@ -142,8 +140,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minX value
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMinXDirAdj() {

        return textPositions.get(0).getXDirAdj();
@@ -157,8 +154,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxX value
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMaxXDirAdj() {

        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
@@ -172,8 +168,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMinYDirAdj() {

        return textPositions.get(0).getYDirAdj() - getTextHeight();
@@ -187,8 +182,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMaxYDirAdj() {

        return textPositions.get(0).getYDirAdj();
@@ -196,42 +190,38 @@ public class TextPositionSequence implements CharSequence {
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getHeight() {

        return getMaxYDirAdj() - getMinYDirAdj();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getWidth() {

        return getMaxXDirAdj() - getMinXDirAdj();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public String getFont() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "none";
+        }
        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public String getFontStyle() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "standard";
+        }
        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);

        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
@@ -243,20 +233,15 @@ public class TextPositionSequence implements CharSequence {
        } else {
            return "standard";
        }
-
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getFontSize() {

        return textPositions.get(0).getFontSizeInPt();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getSpaceWidth() {

        return textPositions.get(0).getWidthOfSpace();
@@ -272,8 +257,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return bounding box of the word in Pdf Coordinate System
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    @SneakyThrows
    public Rectangle getRectangle() {

@@ -8,6 +8,7 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.PageInfo;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;

@@ -24,24 +25,26 @@ public class CvTableParsingAdapter {
        Map<Integer, List<TableCells>> tableCells = new HashMap<>();
        tableServiceResponse.getData()
                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells())));
+                        .addAll(convertTableCells(tableData.getTableCells(), tableData.getPageInfo())));

        return tableCells;
    }


-    private Collection<TableCells> convertTableCells(List<TableCells> tableCells) {
+    private Collection<TableCells> convertTableCells(List<TableCells> tableCells, PageInfo pageInfo) {

        List<TableCells> cvParsedTableCells = new ArrayList<>();

-        tableCells.forEach(t -> cvParsedTableCells.add(TableCells.builder()
-                .y0(t.getY0())
-                .x1(t.getX1())
-                .y1(t.getY1())
-                .x0(t.getX0())
-                .width(t.getWidth())
-                .height(t.getHeight())
-                .build()));
+        tableCells.stream()
+                .filter(cell -> cell.getWidth() < pageInfo.getWidth() * 0.98 && cell.getHeight() < pageInfo.getHeight() * 0.98)
+                .forEach(t -> cvParsedTableCells.add(TableCells.builder()
+                        .y0(t.getY0())
+                        .x1(t.getX1())
+                        .y1(t.getY1())
+                        .x0(t.getX0())
+                        .width(t.getWidth())
+                        .height(t.getHeight())
+                        .build()));

        return cvParsedTableCells;
    }
@@ -12,9 +12,9 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;

 import lombok.RequiredArgsConstructor;
@@ -25,10 +25,13 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RulingCleaningService {

-    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
+    private static final float THRESHOLD = 6;
+
+
+    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {

        if (!rulings.isEmpty()) {
-            snapPoints(rulings, minCharWidth, maxCharHeight);
+            snapPoints(rulings);
        }

        List<Ruling> vrs = new ArrayList<>();
@@ -53,14 +56,11 @@ public class RulingCleaningService {
        }
        List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);

-        return CleanRulings.builder()
-                .vertical(verticalRulingLines)
-                .horizontal(horizontalRulingLines)
-                .build();
+        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
    }


-    public void snapPoints(List<? extends Line2D.Float> rulings, float xThreshold, float yThreshold) {
+    public void snapPoints(List<? extends Line2D.Float> rulings) {

        // collect points and keep a Line -> p1,p2 map
        Map<Line2D.Float, Point2D[]> linesToPoints = new HashMap<>();
@@ -81,7 +81,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < xThreshold) {
+            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -108,7 +108,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < yThreshold) {
+            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -12,7 +12,6 @@ import java.util.Set;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -66,6 +65,17 @@ public class TableExtractionService {
    };


+    public boolean contains(Cell cell, double x, double y, double w, double h) {
+
+        if (cell.isEmpty() || w <= 0 || h <= 0) {
+            return false;
+        }
+        double x0 = cell.getX();
+        double y0 = cell.getY();
+        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
+    }
+
+
    /**
     * Finds tables on a page and moves textblocks into cells of the found tables.
     * Note: This algorithm uses Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -79,16 +89,17 @@ public class TableExtractionService {
     * @param cleanRulings The lines used to build the table.
     * @param page         Page object that contains textblocks and statistics.
     */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page, LayoutParsingType layoutParsingType) {
+    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {

-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical(), layoutParsingType);
+        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());

        List<TextPageBlock> toBeRemoved = new ArrayList<>();

        for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
            TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
            for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
+                if (cell.hasMinimumSize() && contains(cell,
+                        textBlock.getPdfMinX(),
                        textBlock.getPdfMinY(),
                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -102,7 +113,7 @@ public class TableExtractionService {
        cells = new ArrayList<>(new HashSet<>(cells));
        DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);

-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).toList();
+        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);

        List<TablePageBlock> tables = new ArrayList<>();
        for (Rectangle area : spreadsheetAreas) {
@@ -135,16 +146,14 @@ public class TableExtractionService {
    }


-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, LayoutParsingType layoutParsingType) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

-        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
-            // TODO: breaks some tables, for example "1 Abamectin Prr.pdf" try to fix this upstream in RulingCleaningService
-            for (Ruling r : horizontalRulingLines) {
-                if (r.getX2() < r.getX1()) {
-                    double a = r.getX2();
-                    r.x2 = (float) r.getX1();
-                    r.x1 = (float) a;
-                }
+        // Fix for 211.pdf
+        for (Ruling r : horizontalRulingLines) {
+            if (r.getX2() < r.getX1()) {
+                double a = r.getX2();
+                r.x2 = (float) r.getX1();
+                r.x1 = (float) a;
            }
        }

@@ -133,7 +133,7 @@ public class SearchTextWithTextPositionFactory {

    private static void addTextPositionWithFontType(RedTextPosition currentTextPosition, String fontType, List<Integer> fontTypePositions, int stringIdx) {

-        if (currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
+        if (currentTextPosition.getFontName() != null && currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
            fontTypePositions.add(stringIdx);
        }
    }
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -8,6 +9,7 @@ import java.util.Map;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;

@@ -1,18 +1,34 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.parsing;

-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.color.CMMException;
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.*;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
 import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
 import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
-import org.apache.pdfbox.contentstream.operator.state.*;
+import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
+import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
+import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
+import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
+import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
+import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
+import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSNumber;
@@ -21,11 +37,14 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.text.TextPosition;

-import java.awt.color.CMMException;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;

@Getter
@Slf4j
@@ -36,11 +55,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private final List<Ruling> graphicsPath = new ArrayList<>();
    @Setter
    protected PDPage pdpage;
-    private int minCharWidth;
-    private int maxCharWidth;
-    private int minCharHeight;
-    private int maxCharHeight;
-

    private float path_x;
    private float path_y;
@@ -73,7 +87,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        this.addOperator(new SetFontAndSize(this));
        this.addOperator(new SetLineWidth(this));

-
        addOperator(new BeginMarkedContentSequenceWithProperties(this));
 //        addOperator(new BeginMarkedContentSequence(this));
        addOperator(new EndMarkedContentSequence(this));
@@ -232,33 +245,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
            }

-            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
-            if (charWidth < minCharWidth) {
-                minCharWidth = charWidth;
-            }
-            if (charWidth > maxCharWidth) {
-                maxCharWidth = charWidth;
-            }
-
-            int charHeight = (int) textPositions.get(i).getHeightDir();
-            if (charHeight < minCharHeight) {
-                minCharHeight = charHeight;
-            }
-            if (charWidth > maxCharHeight) {
-                maxCharHeight = charHeight;
-            }
-
            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
                startIndex++;
                continue;
            }

            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj()) {
+            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -266,9 +261,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -278,13 +271,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                    .getUnicode()
                    .equals("\t")) && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {

                    // Remove false sequence ends (whitespaces)
-                    if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                            .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
+                    if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
                        for (TextPosition t : sublist) {
                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                        }
@@ -319,13 +309,29 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    }


+    public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {
+
+        return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
+    }
+
+
+    public boolean checkIfSequenceContainsOnlyWhitespaces(List<TextPosition> sublist) {
+
+        return !(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                .getUnicode()
+                .equals("\t")));
+    }
+
+
+    public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {
+
+        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
+                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
+    }
+
    @Override
    public String getText(PDDocument doc) throws IOException {

-        minCharWidth = Integer.MAX_VALUE;
-        maxCharWidth = 0;
-        minCharHeight = Integer.MAX_VALUE;
-        maxCharHeight = 0;
        textPositionSequences.clear();
        rulings.clear();
        graphicsPath.clear();
@@ -2,12 +2,13 @@ package com.knecon.fforesight.service.layoutparser.processor.services.visualizat

 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
+import java.io.File;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.util.HashSet;
-import java.util.Set;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;

-import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
@@ -31,6 +32,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.visualization.
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;

+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -40,29 +44,31 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class ViewerDocumentService {

-
    private static final String LAYER_NAME = "Layout grid";
    private static final int FONT_SIZE = 10;
    public static final float LINE_WIDTH = 1f;

    private final LayoutGridService layoutGridService;
+    private final ObservationRegistry observationRegistry;


    @SneakyThrows
-    public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream, boolean layerVisibilityDefaultValue) {
+    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
+    public void createViewerDocument(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {

+        Path tmpFile = Files.createTempFile("tmpViewerDocument", ".pdf");
+        PDDocument pdDocument = openPDDocument(originFile);
        LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
-        // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
-        // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
-        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
-        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
+
+        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
        PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);

        for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
-            PDPage pdPage = pdDocument.getPage(pageNumber);

+            PDPage pdPage = pdDocument.getPage(pageNumber);
+//
            AffineTransform textDeRotationMatrix = getTextDeRotationTransform(pdPage);
-            addLayerToPageRessources(pdPage);
+            addLayerToPageResources(pdPage);

            // We need to save the graphics state before, such that our appended content cannot be affected by previous content streams with side effects,
            // e.g. not escaped matrix transformations.
@@ -115,16 +121,48 @@ public class ViewerDocumentService {
                contentStream.restoreGraphicsState();
                contentStream.endMarkedContent();
            }
-            dictionariesToUpdate.add(pdPage.getCOSObject());
-            dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
+
+            if (pageNumber % 500 == 0 && pageNumber != 0) { // re-open document every once in a while to save on RAM
+                log.info("Incremental save after {} pages", pageNumber);
+                observedIncrementalSave(pdDocument, destinationFile);
+                pdDocument.close();
+                Files.copy(destinationFile.toPath(), tmpFile, StandardCopyOption.REPLACE_EXISTING);
+                pdDocument = openPDDocument(tmpFile.toFile());
+                layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
+            }
+
        }
-        dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
-//        dictionariesToUpdate.add(pdDocument.getDocument().getTrailer());
-        pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
+        observedIncrementalSave(pdDocument, destinationFile);
+
+        tmpFile.toFile().delete();
+        pdDocument.close();
    }


-    private static void addLayerToPageRessources(PDPage pdPage) {
+    private static PDDocument openPDDocument(File tmpFile) throws IOException {
+
+        PDDocument pdDocument;
+        pdDocument = Loader.loadPDF(tmpFile);
+        pdDocument.setAllSecurityToBeRemoved(true);
+        return pdDocument;
+    }
+
+
+    @SneakyThrows
+    private void observedIncrementalSave(PDDocument pdDocument, File outputFile) {
+
+        Observation.createNotStarted("ViewerDocumentService", observationRegistry).contextualName("incremental-save").observe(() -> {
+            try {
+                pdDocument.save(outputFile);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+    }
+
+
+    private static void addLayerToPageResources(PDPage pdPage) {

        PDResources resources = pdPage.getResources();
        if (resources == null) {
@@ -145,7 +183,7 @@ public class ViewerDocumentService {
    }


-    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate, boolean layerVisibilityDefaultValue) {
+    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, boolean layerVisibilityDefaultValue) {

        PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
        PDOptionalContentProperties ocprops = catalog.getOCProperties();
@@ -161,7 +199,7 @@ public class ViewerDocumentService {
            ocprops.addGroup(layer);
        }
        ocprops.setGroupEnabled(layer, layerVisibilityDefaultValue);
-        dictionariesToUpdate.add(catalog.getCOSObject());
+//        dictionariesToUpdate.add(catalog.getCOSObject());
        return layer;
    }

@@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
    application
    id("com.knecon.fforesight.java-conventions")
-    id("org.springframework.boot") version "3.1.3"
+    id("org.springframework.boot") version "3.1.5"
    id("io.spring.dependency-management") version "1.1.3"
    id("org.sonarqube") version "4.3.0.3225"
    id("io.freefair.lombok") version "8.2.2"
@@ -12,31 +12,43 @@ plugins {

 description = "layoutparser-service-server"

+configurations {
+    all {
+        exclude(group = "commons-logging", module = "commons-logging")
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
+        exclude(group = "com.iqser.red.commons", module = "logging-commons")
+    }
+}
+
+
 val jacksonVersion = "2.15.2"
 val pdfBoxVersion = "3.0.0"
+val springBootStarterVersion = "3.1.5"

 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.14.0")
-    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
+    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
+    implementation("com.knecon.fforesight:tracing-commons:0.3.0")
+    implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
    implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
+    implementation("net.logstash.logback:logstash-logback-encoder:7.4")

    // for integration testing only
    testImplementation(project(":layoutparser-service-internal-api"))

-    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
+    testImplementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0")
    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
-    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.3")
 }

 /*
@@ -5,6 +5,7 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
@@ -13,8 +14,11 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingService
 import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.aop.ObservedAspect;
+
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
-@Import({MetricsConfiguration.class, StorageAutoConfiguration.class,  LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
+@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

@@ -23,4 +27,11 @@ public class Application {
        SpringApplication.run(Application.class, args);
    }

+
+    @Bean
+    public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
+
+        return new ObservedAspect(observationRegistry);
+    }
+
 }
@@ -24,8 +24,7 @@ public class MessagingConfiguration {
    @Bean
    public Queue layoutparsingResponseQueue() {

-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
    }


@@ -5,10 +5,19 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/

 fforesight.tenants.remote: true

+logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
+
+logging.type: ${LOGGING_TYPE:CONSOLE}
+kubernetes.namespace: ${NAMESPACE:default}
+project.version: 1.0-SNAPSHOT
+
+
 server:
  port: 8080

 spring:
+  application:
+    name: layoutparser-service
  main:
    allow-circular-references: true # FIXME
  rabbitmq:
@@ -32,6 +41,13 @@ management:
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
  endpoints.web.exposure.include: prometheus, health
+  tracing:
+    enabled: ${TRACING_ENABLED:false}
+    sampling:
+      probability: ${TRACING_PROBABILITY:1.0}
+  otlp:
+    tracing:
+      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}


 storage:
@@ -0,0 +1,17 @@
+<configuration>
+
+    <springProperty scope="configuration" name="logType" source="logging.type"/>
+    <springProperty scope="context" name="application.name" source="spring.application.name"/>
+    <springProperty scope="context" name="version" source="project.version"/>
+    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="${logType}"/>
+    </root>
+
+</configuration>
@@ -46,12 +46,11 @@ public class BdrJsonBuildTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(File file) {

-        try (PDDocument pdDocument = Loader.loadPDF(file)) {
-            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
-                    pdDocument,
-                    new ImageServiceResponse(),
-                    new TableServiceResponse()));
-        }
+        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                file,
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                file.toString()));
    }


@@ -6,7 +6,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

-import org.apache.pdfbox.Loader;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -96,9 +95,10 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(pdfFileResource.getFile()),
+                pdfFileResource.getFile(),
                new ImageServiceResponse(),
-                new TableServiceResponse()));
+                new TableServiceResponse(),
+                filePath));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -25,8 +25,8 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
    @SneakyThrows
    public void testLayoutParserEndToEnd() {

-        prepareStorage("files/bdr/btd_19_053_1905391.pdf");
-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.TAAS);
+        prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
    }
@@ -6,6 +6,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;

+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -17,7 +18,9 @@ import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest
 import lombok.SneakyThrows;

 public class DocumentDataTests extends BuildDocumentTest {
+
    @Test
+    @Disabled // This test takes waaaaaay too long, it's ridiculous
    @SneakyThrows
    public void createDocumentDataForAllFiles() {

@@ -36,11 +39,12 @@ public class DocumentDataTests extends BuildDocumentTest {
        for (String pdfFileName : pdfFileNames) {
            System.out.println(pdfFileName);
            DocumentData documentData = DocumentDataMapper.toDocumentData(buildGraph(resource.getFile().toPath().getParent().relativize(Path.of(pdfFileName)).toString()));
-                        File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
+            File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
            outputFile.toPath().getParent().toFile().mkdirs();
            try (var out = new FileOutputStream(outputFile.toString().replace(".pdf", ".json"))) {
                ObjectMapperFactory.create().writeValue(out, documentData);
            }
        }
    }
+
 }
@@ -5,7 +5,6 @@ import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;

-import org.apache.pdfbox.Loader;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;
@@ -56,9 +55,10 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
    private void writeJsons(Path filename) {

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(filename.toFile()),
+                filename.toFile(),
                new ImageServiceResponse(),
-                new TableServiceResponse()));
+                new TableServiceResponse(),
+                filename.toFile().toString()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -47,7 +47,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentTest {
    @Disabled
    public void visualizeCraftedDocument() {

-        String filename = "files/crafted document.pdf";
+        String filename = "files/1 Abamectin_prr.pdf";
        visualizePdf(filename);
    }

@@ -1,10 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

-import java.io.FileOutputStream;
+import java.io.File;
 import java.nio.file.Path;

-import org.apache.pdfbox.Loader;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -19,18 +17,16 @@ import lombok.SneakyThrows;
 public class ViewerDocumentTest extends BuildDocumentTest {

    @Test
-    @Disabled
    @SneakyThrows
    public void testViewerDocument() {

-        LayoutGridService layoutGridService = new LayoutGridService();
-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
-        String fileName = "files/bdr/notMergedParagraphs.pdf";
-        Document document = buildGraph(fileName, LayoutParsingType.TAAS);
+        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
-        try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getFile()); var out = new FileOutputStream(tmpFileName)) {
-            viewerDocumentService.createViewerDocument(pdDocument, document, out, true);
-        }
+        LayoutGridService layoutGridService = new LayoutGridService();
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
+        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
+        var documentFile = new ClassPathResource(fileName).getFile();
+        viewerDocumentService.createViewerDocument(documentFile, document, new File(tmpFileName), true);
    }

 }
@@ -1,5 +1,26 @@
 package com.knecon.fforesight.service.layoutparser.server.segmentation;

+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.awt.geom.Rectangle2D;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -15,19 +36,8 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+
 import lombok.SneakyThrows;
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
-import java.awt.geom.Rectangle2D;
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.assertj.core.api.Assertions.assertThat;

 public class PdfSegmentationServiceTest extends AbstractTest {

@@ -50,12 +60,14 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    private SectionsBuilderService sectionsBuilderService;


-    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+    @SneakyThrows
+    public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {

-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                originDocument,
                new ImageServiceResponse(),
-                new TableServiceResponse());
+                tableServiceResponse,
+                "document");

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -65,6 +77,58 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @SneakyThrows
+    public ClassificationDocument buildClassificationDocument(File originDocument) {
+
+        return buildClassificationDocument(originDocument, new TableServiceResponse());
+    }
+
+
+    @Test
+    public void tablesToHtmlDebugger() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");
+
+    }
+
+
+    @Test
+    public void tablesToHtmlDebuggerWithCVResponse() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
+        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
+        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
+
+        toHtml(document, "/tmp/ScanRotationBorder.html");
+    }
+
+
+    @Test
+    public void testScanRotationBorderIsIgnored() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
+        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
+        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+
+        // Quality of the table parsing is not good, because the file is rotated at scanning.
+        // We only asset that the table border is not the page border.
+        tables.forEach(table -> {
+            assertThat(table.getMinX()).isGreaterThan(75);
+            assertThat(table.getMaxX()).isLessThan(512);
+        });
+    }
+
+
    @Test
    @SneakyThrows
    public void testMapping() {
@@ -92,7 +156,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(table.getColCount()).isEqualTo(6);
@@ -106,7 +170,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -124,7 +188,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(9);
@@ -142,7 +206,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -155,19 +219,36 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


-    @Test
+    @Test // Non-sense test
    public void testDoc56Page170() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 4);

        validateTable(document, 0, 1, 1, 0, 0);
        validateTable(document, 1, 2, 2, 0, 0);
-        validateTable(document, 2, 7, 20, 0, 140);
-        validateTable(document, 3, 8, 31, 0, 170);
+        validateTable(document, 2, 6, 20, 0, 0);
+        validateTable(document, 3, 7, 31, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc211() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/211.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 5, 4, 0, 0);
+        validateTable(document, 1, 5, 15, 14, 0);
+        validateTable(document, 2, 5, 14, 11, 0);
+        validateTable(document, 3, 5, 3, 0, 0);

    }

@@ -177,11 +258,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

-        validateTable(document, 0, 8, 8, 0, 2);
+        validateTable(document, 0, 8, 8, 0, 0);

        List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
                        "Author, date",
@@ -191,18 +272,18 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                        "Method meets analytical validation criteria",
                        "Remarks (in case validation criteria are not met)",
                        "Acceptability of the method"),
-                Arrays.asList("",
+                Arrays.asList("Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
                        "Evans P.G. 2001 TMJ4569B, VV-323245",
                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
-                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
                        "Y",
                        "N/A",
@@ -218,7 +299,9 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        toHtml(document, "/tmp/html.html");

        validateTableSize(document, 4);

@@ -231,17 +314,29 @@ public class PdfSegmentationServiceTest extends AbstractTest {


    @Test
+    @Disabled // FIXME Fake Redactions leads to more cells, no solution for this currently
    public void testDocA20622APartB9Page185() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

-        validateTableSize(document, 2);
+        validateTableSize(document, 1);

-        validateTable(document, 0, 5, 5, 0, 23);
-        validateTable(document, 1, 11, 9, 0, 36);
+        validateTable(document, 0, 7, 4, 0, 0);
+    }

+
+    @Test
+    public void testDocA20622APartB9Page185FixedDoc() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185_fixed.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 7, 4, 0, 0);
    }


@@ -250,7 +345,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izZRMS (CZ) fRR Part B7_Page123.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 6);

@@ -269,7 +364,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04_Page11.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 3);

@@ -285,7 +380,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10_Page532.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 9, 0, 0);
@@ -298,7 +393,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

@@ -312,7 +407,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 6, 7, 0);
@@ -325,10 +420,10 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);
-        validateTable(document, 0, 10, 6, 0, 1);
+        validateTable(document, 0, 10, 6, 0, 0);

    }

@@ -338,7 +433,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);
        validateTable(document, 0, 2, 2, 0, 0);
@@ -353,7 +448,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

@@ -369,7 +464,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

@@ -384,7 +479,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

@@ -399,7 +494,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

@@ -414,7 +509,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 4);

@@ -431,7 +526,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

@@ -446,12 +541,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 2);

-        validateTable(document, 0, 6, 8, 0, 2);
-        validateTable(document, 1, 6, 8, 0, 1);
+        validateTable(document, 0, 6, 8, 0, 0);
+        validateTable(document, 1, 6, 8, 0, 0);

    }

@@ -461,7 +556,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

@@ -475,7 +570,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

        validateTableSize(document, 1);

@@ -484,12 +579,37 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @SneakyThrows
+    private void toHtml(ClassificationDocument document, String filename) {
+
+        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+        StringBuilder sb = new StringBuilder();
+
+        int currentPage = 1;
+        for (var table : tables) {
+            if (currentPage != table.getPage()) {
+                currentPage = table.getPage();
+                sb.append("---------------------- Page ").append(currentPage).append("--------------\n");
+            }
+            sb.append("\n\n");
+            sb.append(table.getTextAsHtml());
+        }
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(Path.of(filename).toFile())) {
+            fileOutputStream.write(sb.toString().getBytes());
+        }
+    }
+
+
    private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {

        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
        List<List<Cell>> rows = table.getRows();
        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();

+        for (List<Cell> row : table.getRows()) {
+            row.forEach(r -> System.out.println(r.toString()));
+        }
        assertThat(emptyCellsFoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);

        assertThat(table.getColCount()).isEqualTo(colCount);
@@ -1,21 +1,39 @@
 package com.knecon.fforesight.service.layoutparser.server.services;

+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;

+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;

-public class RulingCleaningServiceTest {
+public class RulingCleaningServiceTest extends BuildDocumentTest {

    @Test
 //    @Disabled
@@ -25,13 +43,98 @@ public class RulingCleaningServiceTest {
        String fileName = "files/211.pdf";
        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
-        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
-
        RulingCleaningService rulingCleaningService = new RulingCleaningService();
+        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
        List<CleanRulings> cleanRulingsPerPage = new LinkedList<>();
        for (PageContents pageContent : pageContents) {
-            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings(), 8, 20));
+            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings()));
+        }
+
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testTableExtraction() {
+
+        LayoutGridService layoutGridService = new LayoutGridService();
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
+
+        ClassPathResource resource = new ClassPathResource("files");
+        List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
+                .filter(path -> path.getFileName().toString().endsWith(".pdf"))
+                .map(Path::toAbsolutePath)
+                .map(Path::toString)
+                .toList();
+
+        for (int i = 0; i < pdfFileNames.size(); i++) {
+            writeJsons(Path.of(pdfFileNames.get(i)));
        }
    }

+
+    @SneakyThrows
+    private void writeJsons(Path filename) {
+
+        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                filename.toFile(),
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                filename.toFile().toString()));
+        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                filename.toFile(),
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                filename.toFile().toString()));
+        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
+        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
+        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
+            String tmpFileNameBefore = "C:/Users/YANNIK~1/AppData/Local/Temp/before." + filename.getFileName().toString();
+            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
+                PdfDraw.drawDocumentGraph(pdDocument, documentGraphBefore);
+                pdDocument.save(tmpFileNameBefore);
+            }
+            String tmpFileNameAfter = "C:/Users/YANNIK~1/AppData/Local/Temp/after." + filename.getFileName().toString();
+            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
+                PdfDraw.drawDocumentGraph(pdDocument, documentGraphAfter);
+                pdDocument.save(tmpFileNameAfter);
+
+            }
+        }
+    }
+
+
+    @SneakyThrows
+    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2, String pdfName) {
+
+        List listStructure1 = structure1.streamAllEntries()
+                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
+                .map(DocumentStructure.EntryData::getProperties)
+                .map(properties -> {
+                    var builder = Table.builder();
+                    PropertiesMapper.parseTableProperties(properties, builder);
+                    return builder.build();
+                })
+                .toList();
+
+        List listStructure2 = structure2.streamAllEntries()
+                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
+                .map(DocumentStructure.EntryData::getProperties)
+                .map(properties -> {
+                    var builder = Table.builder();
+                    PropertiesMapper.parseTableProperties(properties, builder);
+                    return builder.build();
+                })
+                .toList();
+
+        for (int i = 0; i < listStructure1.size(); i++) {
+            Table tableNode1 = (Table) listStructure1.get(i);
+            Table tableNode2 = (Table) listStructure2.get(i);
+            if (tableNode1.getNumberOfRows() != tableNode2.getNumberOfRows() || tableNode1.getNumberOfCols() != tableNode2.getNumberOfCols()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
 }
@@ -1,5 +1,26 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

+import java.io.InputStream;
+import java.util.Map;
+import java.util.Optional;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
@@ -9,22 +30,8 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorage
 import com.knecon.fforesight.service.layoutparser.server.Application;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
-import lombok.SneakyThrows;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.*;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;

-import java.io.InputStream;
-import java.util.Optional;
+import lombok.SneakyThrows;

@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -100,9 +107,11 @@ public abstract class AbstractTest {
        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
    }

+
    protected LayoutParsingRequest buildDefaultLayoutParsingRequest(LayoutParsingType layoutParsingType) {

        return LayoutParsingRequest.builder()
+                .identifier(Map.of("fileId", "1337"))
                .layoutParsingType(layoutParsingType)
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
@@ -116,6 +125,7 @@ public abstract class AbstractTest {
                .build();
    }

+
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile) {

@@ -152,7 +162,6 @@ public abstract class AbstractTest {
    @ComponentScan("com.knecon.fforesight.service.layoutparser")
    public static class TestConfiguration {

-
        @Bean
        @Primary
        public StorageService inmemoryStorage() {
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;

-import java.io.InputStream;
+import java.io.File;

-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

@@ -25,11 +23,9 @@ public abstract class BuildDocumentTest extends AbstractTest {
    @SneakyThrows
    protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {

-        ClassPathResource fileResource = new ClassPathResource(filename);
+        File fileResource = new ClassPathResource(filename).getFile();
        prepareStorage(filename);
-        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream.readAllBytes())) {
-            return layoutParsingPipeline.parseLayout(layoutParsingType, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse());
-        }
+        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), filename);
    }


@@ -7,6 +7,9 @@ fforesight.tenants.remote: true
 server:
  port: 8080

+logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
+logging.type: ${LOGGING_TYPE:CONSOLE}
+
 spring:
  main:
    allow-circular-references: true # FIXME
@@ -1,16 +0,0 @@
-<Configuration>
-
-    <Appenders>
-        <Console name="CONSOLE" target="SYSTEM_OUT">
-            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
-        </Console>
-    </Appenders>
-
-    <Loggers>
-        <Root level="warn">
-            <AppenderRef ref="CONSOLE"/>
-        </Root>
-        <Logger name="com.iqser" level="info"/>
-    </Loggers>
-
-</Configuration>
@@ -0,0 +1,17 @@
+<configuration>
+
+    <springProperty scope="configuration" name="logType" source="logging.type"/>
+    <springProperty scope="context" name="application.name" source="spring.application.name"/>
+    <springProperty scope="context" name="version" source="project.version"/>
+    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="${logType}"/>
+    </root>
+
+</configuration>
@@ -2,7 +2,14 @@
 dir=${PWD##*/}
 gradle assemble

-buildNumber=${1:-1}
+# Get the current Git branch
+branch=$(git rev-parse --abbrev-ref HEAD)

-gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$USER-$buildNumber
-echo "nexus.knecon.com:5001/red/${dir}-server-v1:$USER-$buildNumber"
+# Get the short commit hash (first 5 characters)
+commit_hash=$(git rev-parse --short=5 HEAD)
+
+# Combine branch and commit hash
+buildName="${USER}-${branch}-${commit_hash}"
+
+gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName --no-build-cache
+echo "nexus.knecon.com:5001/ff/${dir}-service-server:$buildName"
Author	SHA1	Message	Date
Dominique Eifländer	9e854379e7	Merge branch 'RED-1137' into 'main' RED-1137: Do not observe actuator endpoints See merge request fforesight/layout-parser!92	2023-12-20 14:11:31 +01:00
Dominique Eifländer	b779c72041	RED-1137: Do not observe actuator endpoints	2023-12-20 14:05:00 +01:00
Dominique Eifländer	760a809900	Merge branch 'RED-7384' into 'main' RED-7384: fixes for migration See merge request fforesight/layout-parser!91	2023-12-20 12:40:00 +01:00
Kilian Schüttler	ba1c7c07ab	RED-7384: fixes for migration	2023-12-20 12:40:00 +01:00
Dominique Eifländer	ca0cbbcb49	Merge branch 'RED-5223' into 'main' RED-5223: Use tracing-commons from fforesight See merge request fforesight/layout-parser!90	2023-12-13 16:05:01 +01:00
Dominique Eifländer	da2cdc288e	RED-5223: Use tracing-commons from fforesight	2023-12-13 15:31:26 +01:00
Dominique Eifländer	68da328889	Merge branch 'queueHotfix' into 'main' hotfix: removed dlq from response queue to be equal to persistence-service See merge request fforesight/layout-parser!89	2023-12-13 09:54:43 +01:00
Dominique Eifländer	711548d1a7	hotfix: removed dlq from response queue to be equal to persistence-service	2023-12-13 09:47:27 +01:00
Dominique Eifländer	2bddcdafee	Merge branch 'RED-5223' into 'main' RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs See merge request fforesight/layout-parser!88	2023-12-11 15:13:22 +01:00
Dominique Eifländer	750ccf4ce2	RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs	2023-12-11 15:06:23 +01:00
Ali Oezyetimoglu	57b5d3f48e	Merge branch 'RED-7714' into 'main' RED-7715 - Add log4j config to enable switching between json/line logs See merge request fforesight/layout-parser!87	2023-12-06 13:00:28 +01:00
Andrei Isvoran	d8c9659469	RED-7715 - Add log4j config to enable switching between json/line logs	2023-12-06 11:59:42 +02:00
Kilian Schüttler	30f060e36c	Merge branch 'enable-caching' into 'main' enable caching in build See merge request fforesight/layout-parser!84	2023-11-24 10:36:55 +01:00
Kilian Schuettler	53a5824e6c	enable caching in build	2023-11-24 10:24:50 +01:00
Dominique Eifländer	e2bcf971c9	Merge branch 'DM-589' into 'main' DM-589: Filter wrong detected cells that borders from rotation at scanning See merge request fforesight/layout-parser!83	2023-11-20 16:07:27 +01:00
Dominique Eifländer	dacc2f7f43	DM-589: Filter wrong detected cells that borders from rotation at scanning	2023-11-20 15:54:02 +01:00
Dominique Eifländer	144a9591a2	Merge branch 'TAAS-103-hotfix' into 'main' * added back in if statement See merge request fforesight/layout-parser!82	2023-11-16 12:48:48 +01:00
yhampe	207d9dec97	* added back in if statement * removed not needed commentar	2023-11-16 12:40:49 +01:00
Yannik Hampe	09ee90222e	Merge branch 'TAAS-103' into 'main' TAAS-103: Table Detection and rotated text See merge request fforesight/layout-parser!81	2023-11-16 09:13:41 +01:00
yhampe	1316a067fe	* removed double chechking for height of cell	2023-11-16 08:51:12 +01:00
yhampe	e203210ade	* removed not needed properties	2023-11-16 08:23:58 +01:00
yhampe	b25d46291a	* checkstyle	2023-11-16 08:12:47 +01:00
yhampe	84148d3b6e	* fixed tests	2023-11-16 07:51:08 +01:00
Dominique Eifländer	a6ba66b1aa	TAAS-103: Fixed values in wrong cells	2023-11-15 13:36:46 +01:00
yhampe	c3e69b2cdf	* fixed bug with incorrect empty cell count by adding threshhold to cell.contains	2023-11-15 10:44:47 +01:00
yhampe	f69331e7d8	*renamed page to firstPage in DocumentStructure and Table	2023-11-07 10:21:19 +01:00
yhampe	01493dc033	TAAS-103: Table Detection and rotated text * added page property to DocumentStructure to be able to get page of found tables * added a method to TableExtractionService to get the table area * added calculateMinCharWidthAndMaxCharHeightInsideTable to LayoutParsingPipeline to calculate the values based upon table area * refactored PDFLinesTextStripper for better readability *removed textMatrix from RedTextPosition as it is no longer needed	2023-11-07 08:47:28 +01:00
yhampe	459e0c8be7	TAAS-103:	2023-11-07 08:39:15 +01:00
Kilian Schüttler	1b1f777706	Merge branch 'RED-7806' into 'main' RED-7806 - Specific customer document cannot be processed See merge request fforesight/layout-parser!79	2023-10-25 10:50:34 +02:00
Corina Olariu	0e0a811f9d	RED-7806 - Specific customer document cannot be processed - add brackets	2023-10-25 11:36:54 +03:00
Corina Olariu	efa3d75479	RED-7806 - Specific customer document cannot be processed - check for font name null before using to avoid the NPE	2023-10-25 09:16:47 +03:00