Native Build

RED-7461: Fixed pr findings
RED-7461: Use marked content to classify headers and footers if available
2023-08-22 13:39:54 +02:00 · 2023-08-21 16:57:37 +02:00 · 2023-08-21 16:02:24 +02:00 · 2023-08-21 15:31:11 +02:00 · 2023-08-19 09:17:27 +02:00 · 2023-08-19 04:15:48 +00:00
58 changed files with 1302 additions and 394 deletions
@@ -38,3 +38,7 @@ build/

 ### VS Code ###
 .vscode/
+gradlew.bat
+gradlew
+gradle.properties
+gradle/
@@ -1,4 +1,21 @@
 include:
  - project: 'gitlab/gitlab'
    ref: 'main'
-    file: 'ci-templates/gradle_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG
@@ -17,6 +17,7 @@
        <exclude name="AvoidDuplicateLiterals"/>
        <exclude name="AvoidFieldNameMatchingMethodName"/>
        <exclude name="AvoidFieldNameMatchingTypeName"/>
+        <exclude name="TestClassWithoutTestCases"/>
    </rule>

 </ruleset>
@@ -1,6 +1,6 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.1.0"
+    id("io.freefair.lombok") version "8.2.2"
 }

 description = "layoutparser-service-internal-api"
@@ -19,6 +19,7 @@ public record LayoutParsingRequest(
        @NonNull String positionBlockFileStorageId,
        @NonNull String pageFileStorageId,
        @NonNull String simplifiedTextStorageId,
+        @NonNull String viewerDocumentStorageId,
        @NonNull String sectionGridStorageId) {

 }
@@ -1,26 +1,26 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.1.0"
+    id("io.freefair.lombok") version "8.2.2"
 }

 description = "layoutparser-service-processor"

-val jacksonVersion = "2.15.0-rc2"
-val pdfBoxVersion = "3.0.0-alpha2"
+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0-RC1"

 dependencies {
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0") {
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.138.0") {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.iqser.red.commons:storage-commons:2.36.0")

    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.1.2")
 }
@@ -2,33 +2,57 @@ package com.knecon.fforesight.service.layoutparser.processor;

 import static java.lang.String.format;

+import java.awt.geom.Rectangle2D;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
+import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;

 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@@ -39,13 +63,19 @@ public class LayoutParsingPipeline {
    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
    private final CvTableParsingAdapter cvTableParsingAdapter;
    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final PdfParsingService pdfParsingService;
    private final SectionsBuilderService sectionsBuilderService;
    private final SectionGridCreatorService sectionGridCreatorService;
    private final TaasClassificationService taasClassificationService;
    private final RedactManagerClassificationService redactManagerClassificationService;
    private final DocuMineClassificationService docuMineClassificationService;
    private final SimplifiedSectionTextService simplifiedSectionTextService;
+    private final BodyTextFrameService bodyTextFrameService;
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;
+    private final ViewerDocumentService viewerDocumentService;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -63,12 +93,20 @@ public class LayoutParsingPipeline {
                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
            }

-            Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+
            int numberOfPages = originDocument.getNumberOfPages();

            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+            try (var out = new ByteArrayOutputStream()) {
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out);
+                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
+            }
+
            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
@@ -88,15 +126,77 @@ public class LayoutParsingPipeline {
    }


-    public Document parseLayout(LayoutParsingType layoutParsingType,
-                                PDDocument originDocument,
-                                ImageServiceResponse imageServiceResponse,
-                                TableServiceResponse tableServiceResponse) {
+    @SneakyThrows
+    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
+                                              PDDocument originDocument,
+                                              ImageServiceResponse imageServiceResponse,
+                                              TableServiceResponse tableServiceResponse) {

-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
-                originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
+        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
+        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
+
+        ClassificationDocument classificationDocument = new ClassificationDocument();
+        List<ClassificationPage> classificationPages = new ArrayList<>();
+
+        originDocument.setAllSecurityToBeRemoved(true);
+        long pageCount = originDocument.getNumberOfPages();
+
+        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+            classificationDocument.setPages(classificationPages);
+            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+            PDPage pdPage = originDocument.getPage(pageNumber - 1);
+            stripper.setPageNumber(pageNumber);
+            stripper.setStartPage(pageNumber);
+            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE)) {
+                stripper.setSortByPosition(true);
+            }
+            stripper.getText(originDocument);
+
+            PDRectangle pdr = pdPage.getMediaBox();
+
+            int rotation = pdPage.getRotation();
+            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+            PDRectangle cropbox = pdPage.getCropBox();
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                    stripper.getRulings(),
+                    stripper.getMinCharWidth(),
+                    stripper.getMaxCharHeight());
+
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER ->
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS ->
+                        taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE ->
+                        docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            };
+            classificationPage.setCleanRulings(cleanRulings);
+            classificationPage.setRotation(rotation);
+            classificationPage.setLandscape(isLandscape);
+            classificationPage.setPageNumber(pageNumber);
+            classificationPage.setPageWidth(cropbox.getWidth());
+            classificationPage.setPageHeight(cropbox.getHeight());
+
+            // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
+            classificationPage.setMarkedContentBboxPerType(convertMarkedContents(stripper.getMarkedContents()));
+
+            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+                classificationPage.setImages(pdfImages.get(pageNumber));
+                imageServiceResponseAdapter.findOcr(classificationPage);
+            }
+
+            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            buildPageStatistics(classificationPage);
+            increaseDocumentStatistics(classificationPage, classificationDocument);
+
+            classificationPages.add(classificationPage);
+        }
+
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);

        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
@@ -107,40 +207,45 @@ public class LayoutParsingPipeline {
        sectionsBuilderService.buildSections(classificationDocument);
        sectionsBuilderService.addImagesToSections(classificationDocument);

-        return DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+        return classificationDocument;
    }


-    public Document parseLayoutWithTimer(LayoutParsingType layoutParsingType,
-                                         PDDocument originDocument,
-                                         ImageServiceResponse imageServiceResponse,
-                                         TableServiceResponse tableServiceResponse) {
+    private Map<String, Rectangle2D> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+        Map<String, Rectangle2D> markedContentBboxes = new HashMap<>();
+        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
+        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
+        return markedContentBboxes;
+    }

-        long start = System.currentTimeMillis();
+    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {

-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
-                originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
-
-        System.out.printf("parsed %d ms", System.currentTimeMillis() - start);
-
-        start = System.currentTimeMillis();
-        switch (layoutParsingType) {
-            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
-            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
+        if (!classificationPage.isLandscape()) {
+            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
        }
-        System.out.printf(", classified %d ms", System.currentTimeMillis() - start);
+        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
+        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
+        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
+    }

-        start = System.currentTimeMillis();
-        sectionsBuilderService.buildSections(classificationDocument);
-        System.out.printf(", sections built %d ms", System.currentTimeMillis() - start);

-        start = System.currentTimeMillis();
-        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-        System.out.printf(", graph constructed %d ms", System.currentTimeMillis() - start);
-        return document;
+    private void buildPageStatistics(ClassificationPage classificationPage) {
+
+        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
+        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
+                    continue;
+                }
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
+                    classificationPage.getFontCounter().add(word.getFont());
+                    classificationPage.getFontSizeCounter().add(word.getFontSize());
+                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
+                }
+            }
+        }
+
    }

 }
@@ -1,10 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor;

+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;

 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
@@ -16,14 +21,15 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@@ -37,10 +43,11 @@ public class LayoutParsingStorageService {

    public PDDocument getOriginFile(String storageId) throws IOException {

-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var originDocumentInputStream = getObject(storageId)) {
            File tempFile = createTempFile("document", ".pdf");
            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
            }
            return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
        }
@@ -49,19 +56,22 @@ public class LayoutParsingStorageService {

    public ImageServiceResponse getImagesFile(String storageId) throws IOException {

-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (InputStream inputStream = getObject(storageId)) {

-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
        }
    }


    public TableServiceResponse getTablesFile(String storageId) throws IOException {

-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+        try (var tableClassificationStream = getObject(storageId)) {

+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
        }
    }

@@ -117,4 +127,24 @@ public class LayoutParsingStorageService {
        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
    }

+
+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
+
+    @SneakyThrows
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {
+
+        try (var in = new ByteArrayInputStream(out.toByteArray())) {
+
+            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
+        }
+    }
+
 }
@@ -1,15 +1,20 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;

+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;

 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;

@Data
@RequiredArgsConstructor
@@ -35,4 +40,8 @@ public class ClassificationPage {
    private float pageWidth;
    private float pageHeight;

+    CleanRulings cleanRulings;
+
+    private Map<String, Rectangle2D> markedContentBboxPerType = new HashMap<>();
+
 }
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;

+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

 import lombok.AllArgsConstructor;
@@ -17,5 +18,5 @@ public class PageContents {
    List<TextPositionSequence> sortedTextPositionSequences;
    Rectangle2D cropBox;
    Rectangle2D mediaBox;
-
+    List<Ruling> rulings;
 }
@@ -28,6 +28,7 @@ public class Ruling extends Line2D.Float {
        super(p1, p2);
    }

+
    public Ruling straightenVertical() {

        double y1 = Math.min(getY1(), getY2());
@@ -36,6 +37,7 @@ public class Ruling extends Line2D.Float {
        return new Ruling(new Point2D.Double(x, y1), new Point2D.Double(x, y2));
    }

+
    public Ruling straightenHorizontal() {

        double x1 = Math.min(getX1(), getX2());
@@ -444,6 +446,16 @@ public class Ruling extends Line2D.Float {
    }


+    public boolean almostMatches(Ruling ruling) {
+
+        final float TOLERANCE = 1;
+        return Math.abs(ruling.getX1() - x1) < TOLERANCE &&//
+                Math.abs(ruling.getY1() - y1) < TOLERANCE &&//
+                Math.abs(ruling.getX2() - x2) < TOLERANCE &&//
+                Math.abs(ruling.getY2() - y2) < TOLERANCE;
+    }
+
+
    private enum SOType {
        VERTICAL,
        HRIGHT,
@@ -73,6 +73,7 @@ public class TextPageBlock extends AbstractPageBlock {

        return sequences.get(0).getPageWidth();
    }
+    

    public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Line2D;
+
+public record ColoredLine(Line2D line, Color color) {
+
+}
@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
+
+public record ColoredRectangle(Rectangle2D rectangle2D, Color color) {
+
+}
@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
+
+public record FilledRectangle(Rectangle2D rectangle2D, Color color, float alpha) {
+
+}
@@ -0,0 +1,27 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class LayoutGrid {
+
+    int numberOfPages;
+    List<VisualizationsOnPage> visualizationsPerPages;
+
+
+    public LayoutGrid(int numberOfPages) {
+
+        this.numberOfPages = numberOfPages;
+        this.visualizationsPerPages = new ArrayList<>(numberOfPages);
+        for (int i = 0; i < numberOfPages; i++) {
+            this.visualizationsPerPages.add(VisualizationsOnPage.builder().pageNumber(i).build());
+        }
+    }
+
+}
@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.geom.Point2D;
+
+public record PlacedText(String text, Point2D lineStart) {
+
+}
@@ -0,0 +1,26 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@Builder
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class VisualizationsOnPage {
+
+    int pageNumber;
+    @Builder.Default
+    List<PlacedText> placedTexts = new LinkedList<>();
+    @Builder.Default
+    List<ColoredLine> coloredLines = new LinkedList<>();
+    @Builder.Default
+    List<ColoredRectangle> coloredRectangles = new LinkedList<>();
+    @Builder.Default
+    List<FilledRectangle> filledRectangles = new LinkedList<>();
+
+}
@@ -2,12 +2,14 @@ package com.knecon.fforesight.service.layoutparser.processor.services;

 import java.util.List;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -19,6 +21,64 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 public class BodyTextFrameService {


+    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {
+
+        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
+        Rectangle landscapeBodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), true, layoutParsingType);
+        for (ClassificationPage page : classificationDocument.getPages()) {
+            setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
+        }
+    }
+
+
+/*
+    private Rectangle calculateBodyTextFrameByRulings(List<ClassificationPage> pages) {
+
+        Map<ClassificationPage, List<Ruling>> potentialHeaderRulingsPerPage = new HashMap<>();
+        Map<ClassificationPage, List<Ruling>> potentialFooterRulingsPerPage = new HashMap<>();
+
+        for (var page : pages) {
+            potentialHeaderRulingsPerPage.put(page,
+                    page.getCleanRulings()
+                            .getHorizontal()
+                            .stream()
+                            .filter(ruling -> ruling.getY1() > page.getPageHeight() * 0.8)
+                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
+                            .toList());
+            potentialFooterRulingsPerPage.put(page,
+                    page.getCleanRulings()
+                            .getHorizontal()
+                            .stream()
+                            .filter(ruling -> ruling.getY1() < page.getPageHeight() * 0.2)
+                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
+                            .toList());
+        }
+
+        Optional<Ruling> headerRuling = potentialHeaderRulingsPerPage.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(ruling -> potentialHeaderRulingsPerPage.values()
+                        .stream()
+                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
+                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
+                .min(Comparator.comparingDouble(Ruling::getY1));
+
+        Optional<Ruling> footerRuling = potentialFooterRulingsPerPage.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(ruling -> potentialHeaderRulingsPerPage.values()
+                        .stream()
+                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
+                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
+                .max(Comparator.comparingDouble(Ruling::getY1));
+
+        double maxY = headerRuling.isPresent() ? headerRuling.get().y1 : pages.stream().mapToDouble(ClassificationPage::getPageHeight).max().orElse(Double.MAX_VALUE);
+        double minY = footerRuling.map(ruling -> ruling.y1).orElse(0F);
+        double maxX = pages.stream().mapToDouble(ClassificationPage::getPageWidth).max().orElse(Double.MAX_VALUE);
+
+        return new Rectangle(new Point((float) maxX, (float) maxY), (float) 0, (float) minY, -1);
+    }
+*/


    /**
@@ -34,7 +94,7 @@ public class BodyTextFrameService {
     * @param bodyTextFrame          frame that contains the main text on portrait pages
     * @param landscapeBodyTextFrame frame that contains the main text on landscape pages
     */
-    public void setBodyTextFrameAdjustedToPage(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
+    private void setBodyTextFrameAdjustedToPage(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {

        Rectangle textFrame = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;

@@ -69,7 +129,10 @@ public class BodyTextFrameService {
     * @param landscape               Calculate for landscape or portrait
     * @return Rectangle of the text frame
     */
-    public Rectangle calculateBodyTextFrame(List<ClassificationPage> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape, LayoutParsingType layoutParsingType) {
+    private Rectangle calculateBodyTextFrame(List<ClassificationPage> pages,
+                                             FloatFrequencyCounter documentFontSizeCounter,
+                                             boolean landscape,
+                                             LayoutParsingType layoutParsingType) {

        float approximateHeaderLineCount;
        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
@@ -94,9 +157,14 @@ public class BodyTextFrameService {
                        continue;
                    }

+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                        continue;
+                    }
+
                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10)
-                            || !layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount){
+                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10) || !layoutParsingType.equals(
+                            LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount) {
                        continue;
                    }

@@ -170,4 +238,4 @@ public class BodyTextFrameService {

    }

-}
+}
@@ -21,9 +21,9 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTrans
 import lombok.experimental.UtilityClass;

@UtilityClass
-public class TextPositionSequenceSorter {
+public class PageContentExtractor {

-    public List<PageContents> getSortedTextPositionsWithPages(String filename) throws IOException {
+    public List<PageContents> getSortedPageContents(String filename) throws IOException {

        List<PageContents> textPositionSequencesPerPage = new LinkedList<>();
        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
@@ -49,7 +49,8 @@ public class TextPositionSequenceSorter {

                    textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
                            RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
-                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox())));
+                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox()),
+                            stripper.getRulings()));
                }
            }
        }
@@ -1,151 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
-
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class PdfParsingService {
-
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final TaasBlockificationService taasBlockificationService;
-    private final DocuMineBlockificationService docuMineBlockificationService;
-    private final RedactManagerBlockificationService redactManagerBlockificationService;
-
-
-    public ClassificationDocument parseDocument(LayoutParsingType layoutParsingType,
-                                                PDDocument originDocument,
-                                                Map<Integer, List<TableCells>> pdfTableCells,
-                                                Map<Integer, List<ClassifiedImage>> pdfImages) {
-
-        ClassificationDocument document = new ClassificationDocument();
-        List<ClassificationPage> classificationPages = new ArrayList<>();
-
-        originDocument.setAllSecurityToBeRemoved(true);
-        long pageCount = originDocument.getNumberOfPages();
-
-        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-            parsePage(layoutParsingType, pdfImages, originDocument, pdfTableCells, document, classificationPages, pageNumber);
-        }
-
-        document.setPages(classificationPages);
-
-        return document;
-    }
-
-
-    @SneakyThrows
-    private void parsePage(LayoutParsingType layoutParsingType,
-                           Map<Integer, List<ClassifiedImage>> pdfImages,
-                           PDDocument pdDocument,
-                           Map<Integer, List<TableCells>> pdfTableCells,
-                           ClassificationDocument document,
-                           List<ClassificationPage> classificationPages,
-                           int pageNumber) {
-
-        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-        stripper.setPageNumber(pageNumber);
-        stripper.setStartPage(pageNumber);
-        stripper.setEndPage(pageNumber);
-        stripper.setPdpage(pdPage);
-        stripper.getText(pdDocument);
-
-        PDRectangle pdr = pdPage.getMediaBox();
-
-        int rotation = pdPage.getRotation();
-        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-        PDRectangle cropbox = pdPage.getCropBox();
-        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                stripper.getRulings(),
-                stripper.getMinCharWidth(),
-                stripper.getMaxCharHeight());
-
-        ClassificationPage classificationPage = switch (layoutParsingType) {
-            case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-        };
-
-        classificationPage.setRotation(rotation);
-        classificationPage.setLandscape(isLandscape);
-        classificationPage.setPageNumber(pageNumber);
-        classificationPage.setPageWidth(cropbox.getWidth());
-        classificationPage.setPageHeight(cropbox.getHeight());
-
-        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-            classificationPage.setImages(pdfImages.get(pageNumber));
-            imageServiceResponseAdapter.findOcr(classificationPage);
-        }
-
-        tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
-        buildPageStatistics(classificationPage);
-        increaseDocumentStatistics(classificationPage, document);
-
-        classificationPages.add(classificationPage);
-    }
-
-
-    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
-
-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-        }
-        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
-        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
-        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
-    }
-
-
-    private void buildPageStatistics(ClassificationPage classificationPage) {
-
-        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
-        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
-            if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getSequences() == null) {
-                    continue;
-                }
-                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
-                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
-                    classificationPage.getFontCounter().add(word.getFont());
-                    classificationPage.getFontSizeCounter().add(word.getFontSize());
-                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
-                }
-            }
-        }
-
-    }
-
-}
-
-
@@ -5,6 +5,9 @@ import static java.util.stream.Collectors.toSet;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

@@ -23,6 +26,8 @@ public class DocuMineBlockificationService {

    static final float THRESHOLD = 1f;

+    Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);
+

    /**
     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
@@ -43,7 +48,6 @@ public class DocuMineBlockificationService {
        float maxX = 0;
        float minY = 1000;
        float maxY = 0;
-
        TextPositionSequence prev = null;

        boolean wasSplitted = false;
@@ -60,7 +64,10 @@ public class DocuMineBlockificationService {
            boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle()
                    .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));

-            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap)) {
+            Matcher matcher = pattern.matcher(chunkWords.stream().collect(Collectors.joining(" ")).toString());
+            boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();
+
+            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap || startsOnSameX)) {

                Orientation prevOrientation = null;
                if (!chunkBlockList1.isEmpty()) {
@@ -231,3 +238,4 @@ public class DocuMineBlockificationService {
    }

 }
+
@@ -5,43 +5,36 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

-
@Slf4j
@Service
@RequiredArgsConstructor
 public class DocuMineClassificationService {

-    private final BodyTextFrameService bodyTextFrameService;
-    private static final Pattern pattern = Pattern.compile("^(\\d{1,1}\\.?){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z\\[\\]\\-]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern2 = Pattern.compile(".*\\d{4}$", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern2 = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
    private static final Pattern pattern3 = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");


    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.DOCUMINE);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.DOCUMINE);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
-            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
            classifyPage(page, document, headlineFontSizes);
        }
    }
@@ -70,12 +63,16 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
+        ) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
+        ) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -115,4 +112,4 @@ public class DocuMineClassificationService {
        }
    }

-}
+}
@@ -3,16 +3,14 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -23,19 +21,14 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RedactManagerClassificationService {

-    private final BodyTextFrameService bodyTextFrameService;
-

    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.REDACT_MANAGER);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.REDACT_MANAGER);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
-            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
            classifyPage(page, document, headlineFontSizes);
        }
    }
@@ -59,11 +52,13 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
@@ -3,10 +3,9 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -28,14 +27,13 @@ public class TaasClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.TAAS);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.TAAS);
+
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
-            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
+
            classifyPage(page, document, headlineFontSizes);
        }
    }
@@ -59,9 +57,11 @@ public class TaasClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
            textBlock.setClassification(PageBlockType.HEADER);
-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -32,7 +32,12 @@ public class TableNodeFactory {
        setPageNumberInCells(tablesToMerge);
        Set<Page> pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet());
        List<List<Cell>> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList();
-        Table table = Table.builder().documentTree(context.getDocumentTree()).numberOfCols(mergedRows.get(0).size()).numberOfRows(mergedRows.size()).build();
+
+        Table table = Table.builder()
+                .documentTree(context.getDocumentTree())
+                .numberOfCols(mergedRows.isEmpty() ? 0 : mergedRows.get(0).size())
+                .numberOfRows(mergedRows.size())
+                .build();

        pages.forEach(page -> addTableToPage(page, parentNode, table));

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
@@ -87,7 +88,8 @@ public class PropertiesMapper {

    public static String toString(Rectangle2D rectangle2D) {

-        return String.format("%f%s%f%s%f%s%f",
+        return String.format(Locale.US,
+                "%f%s%f%s%f%s%f",
                rectangle2D.getX(),
                DocumentStructure.RECTANGLE_DELIMITER,
                rectangle2D.getY(),
@@ -1,32 +1,18 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.parsing;

-import java.awt.color.CMMException;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
-import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
-import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
-import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
-import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
+import org.apache.pdfbox.contentstream.operator.color.*;
+import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
+import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
+import org.apache.pdfbox.contentstream.operator.state.*;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSNumber;
@@ -35,14 +21,11 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.text.TextPosition;

-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-
-import lombok.Getter;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.color.CMMException;
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;

@Getter
@Slf4j
@@ -58,6 +41,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private int minCharHeight;
    private int maxCharHeight;

+
    private float path_x;
    private float path_y;

@@ -88,6 +72,12 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        this.addOperator(new SetNonStrokingColorN());
        this.addOperator(new SetFontAndSize());
        this.addOperator(new SetLineWidth());
+
+
+        addOperator(new BeginMarkedContentSequenceWithProperties());
+//        addOperator(new BeginMarkedContentSequence());
+        addOperator(new EndMarkedContentSequence());
+
    }


@@ -98,6 +88,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

        //move
        switch (operation) {
+
            case OperatorName.MOVE_TO:
                if (arguments.size() == 2) {
                    Point2D.Float pos = transformPosition(floatValue(arguments.get(0)), floatValue(arguments.get(1)));
@@ -219,7 +210,11 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            // This is a quick and dirt hack
            // Happens for file 216.pdf
            log.debug(e.getMessage());
-            return color.getComponents()[0] == 0 && color.getComponents()[1] == 0 && color.getComponents()[2] == 0 && color.getComponents()[1] == 1;
+            var result = true;
+            for (var component : color.getComponents()) {
+                result = result && component == 0;
+            }
+            return result;
        }
    }

@@ -344,3 +339,4 @@ public class PDFLinesTextStripper extends PDFTextStripper {

 }

+
@@ -25,27 +25,20 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.regex.Pattern;

+import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageTree;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.pdmodel.graphics.PDXObject;
 import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
 import org.apache.pdfbox.pdmodel.interactive.pagenavigation.PDThreadBead;
 import org.apache.pdfbox.text.TextPosition;
@@ -63,6 +56,10 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    private static float defaultDropThreshold = 2.5f;
    private static final boolean useCustomQuickSort;

+    @Getter
+    protected final List<PDMarkedContent> markedContents = new ArrayList<>();
+    protected final Deque<PDMarkedContent> currentMarkedContents = new ArrayDeque<>();
+
    private static final Log LOG = LogFactory.getLog(PDFTextStripper.class);

    // enable the ability to set the default indent/drop thresholds
@@ -196,6 +193,45 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    }


+
+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
+    {
+        PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
+        if (this.currentMarkedContents.isEmpty())
+        {
+            this.markedContents.add(markedContent);
+        }
+        else
+        {
+            PDMarkedContent currentMarkedContent =
+                    this.currentMarkedContents.peek();
+            if (currentMarkedContent != null)
+            {
+                currentMarkedContent.addMarkedContent(markedContent);
+            }
+        }
+        this.currentMarkedContents.push(markedContent);
+    }
+
+    @Override
+    public void endMarkedContentSequence()
+    {
+        if (!this.currentMarkedContents.isEmpty())
+        {
+            this.currentMarkedContents.pop();
+        }
+    }
+
+
+    public void xobject(PDXObject xobject)
+    {
+        if (!this.currentMarkedContents.isEmpty())
+        {
+            this.currentMarkedContents.peek().addXObject(xobject);
+        }
+    }
+
+
    /**
     * This will return the text of a document. See writeText. <br>
     * NOTE: The document must not be encrypted when coming into this method.
@@ -877,7 +913,12 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    textList.add(text);
                }
            }
+            if (!this.currentMarkedContents.isEmpty())
+            {
+                this.currentMarkedContents.peek().addText(text);
+            }
        }
+
    }


@@ -2103,6 +2144,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            this.isHangingIndent = true;
        }

+
    }

 }
@@ -0,0 +1,282 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
+
+import static com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService.LINE_WIDTH;
+
+import java.awt.Color;
+import java.awt.geom.Line2D;
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.ColoredLine;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.ColoredRectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.LayoutGrid;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+@Service
+public class LayoutGridService {
+
+    private static final Color INNER_LINES_COLOR = new Color(255, 175, 175);
+    private static final Color PARAGRAPH_COLOR = new Color(70, 130, 180);
+    public static final Color TABLE_COLOR = new Color(102, 205, 170);
+    public static final Color SECTION_COLOR = new Color(50, 50, 50);
+    public static final Color HEADLINE_COLOR = new Color(162, 56, 56);
+    public static final Color HEADER_COLOR = new Color(171, 131, 6);
+    public static final Color IMAGE_COLOR = new Color(253, 63, 146);
+
+
+    public LayoutGrid createLayoutGrid(Document document) {
+
+        LayoutGrid layoutGrid = new LayoutGrid(document.getNumberOfPages());
+        document.streamAllSubNodes().forEach(semanticNode -> {
+            Color color = switch (semanticNode.getType()) {
+                case PARAGRAPH -> PARAGRAPH_COLOR;
+                case TABLE -> TABLE_COLOR;
+                case SECTION -> SECTION_COLOR;
+                case HEADLINE -> HEADLINE_COLOR;
+                case HEADER, FOOTER -> HEADER_COLOR;
+                case IMAGE -> IMAGE_COLOR;
+                default -> null;
+            };
+            if (isNotSectionOrTableCellOrDocument(semanticNode)) {
+                addAsRectangle(semanticNode, layoutGrid, color);
+            }
+            if (semanticNode.getType().equals(NodeType.SECTION)) {
+                addSection(semanticNode, layoutGrid, color);
+            }
+            if (semanticNode.getType().equals(NodeType.TABLE)) {
+                Table table = (Table) semanticNode;
+                addInnerTableLines(table, layoutGrid);
+            }
+        });
+
+        return layoutGrid;
+    }
+
+
+    private void addInnerTableLines(Table table, LayoutGrid layoutGrid) {
+
+        if (table.getNumberOfCols() < 1 || table.getNumberOfRows() < 1) {
+            return;
+        }
+        for (Page page : table.getPages()) {
+            Optional<Integer> optionalFirstRowOnPage = table.streamCol(0).filter(tableCell -> tableCell.isOnPage(page.getNumber())).map(TableCell::getRow).findFirst();
+            if (optionalFirstRowOnPage.isEmpty()) {
+                continue;
+            }
+            int firstRowOnPage = optionalFirstRowOnPage.get();
+            Stream<Double> xStream = switch (page.getRotation()) {
+                case 90 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinX);
+                case 180 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMaxX);
+                case 270 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMaxX);
+                default -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMinX);
+            };
+            List<Double> xs = xStream.collect(Collectors.toList());
+            xs.remove(0);
+            Stream<Double> yStream = switch (page.getRotation()) {
+                case 90 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMinY);
+                case 180 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinY);
+                case 270 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMaxY);
+                default -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMaxY);
+            };
+            List<Double> ys = yStream.collect(Collectors.toList());
+            ys.remove(0);
+
+            Rectangle2D tableBBox = table.getBBox().get(table.getFirstPage());
+            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredLines();
+            xs.forEach(x -> {
+                Line2D line = new Line2D.Double(new Point2D.Double(x, tableBBox.getMaxY()), new Point2D.Double(x, tableBBox.getMinY()));
+                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR));
+            });
+            ys.forEach(y -> {
+                Line2D line = new Line2D.Double(new Point2D.Double(tableBBox.getMinX(), y), new Point2D.Double(tableBBox.getMaxX(), y));
+                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR));
+            });
+        }
+    }
+
+
+    private static Stream<Rectangle2D> streamBBoxOfCellsOnPage(Stream<TableCell> table, Page page) {
+
+        return table.filter(tableCell -> tableCell.isOnPage(page.getNumber())).map(TableCell::getBBox).map(bBoxMap -> bBoxMap.get(page));
+    }
+
+
+    private void addSection(SemanticNode semanticNode, LayoutGrid layoutGrid, Color color) {
+
+        Map<Page, Rectangle2D> bBoxMap = semanticNode.getBBox();
+        List<SemanticNode> subSections = semanticNode.streamAllSubNodesOfType(NodeType.SECTION).toList();
+        Page firstPage = semanticNode.getFirstPage();
+        if (!subSections.isEmpty()) {
+            addPlacedText(firstPage, bBoxMap.get(firstPage), buildTreeIdString(semanticNode), layoutGrid);
+        } else {
+            bBoxMap.forEach(((page, textBBox) -> addPlacedText(page, textBBox, buildTreeIdString(semanticNode), layoutGrid)));
+        }
+        if (bBoxMap.values().size() == 1) {
+            Rectangle2D r = RectangleTransformations.pad(bBoxMap.get(firstPage), LINE_WIDTH, LINE_WIDTH);
+            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(firstPage.getNumber() - 1).getColoredLines();
+            List<Line2D> lines = createLinesFromRectangle(r, firstPage.getRotation());
+            // add string to top line
+            var firstLine = lines.remove(0);
+            coloredLines.add(new ColoredLine(firstLine, color));
+            for (Line2D line : lines) {
+                coloredLines.add(new ColoredLine(line, color));
+            }
+            return;
+        }
+        List<Page> pagesInOrder = bBoxMap.keySet().stream().sorted(Comparator.comparingInt(Page::getNumber)).collect(Collectors.toList());
+        pagesInOrder.remove(0);
+        addLinesForFirstPageOfSection(semanticNode, color, firstPage, layoutGrid);
+        var lastPage = pagesInOrder.remove(pagesInOrder.size() - 1);
+        addLinesForLastPageOfSection(semanticNode, color, lastPage, layoutGrid);
+        for (Page middlePage : pagesInOrder) {
+            addLinesForMiddlePageOfSection(semanticNode, color, middlePage, layoutGrid);
+        }
+    }
+
+
+    private void addPlacedText(Page page, Rectangle2D textBBox, String s, LayoutGrid layoutGrid) {
+
+        Point2D.Float upperLeftCorner = switch (page.getRotation()) {
+            case 90 -> new Point2D.Float((float) (textBBox.getMinX()), (float) textBBox.getMinY());
+            case 180 -> new Point2D.Float((float) (textBBox.getMaxX()), (float) textBBox.getMinY());
+            case 270 -> new Point2D.Float((float) (textBBox.getMaxX()), (float) textBBox.getMaxY());
+            default -> new Point2D.Float((float) (textBBox.getMinX()), (float) textBBox.getMaxY());
+        };
+        var placedTexts = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getPlacedTexts();
+        placedTexts.add(new PlacedText(s, upperLeftCorner));
+    }
+
+
+    private void addLinesForMiddlePageOfSection(SemanticNode semanticNode, Color color, Page middlePage, LayoutGrid layoutGrid) {
+
+        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(middlePage.getNumber() - 1).getColoredLines();
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(middlePage), LINE_WIDTH, LINE_WIDTH);
+        var midPageLines = createLinesFromRectangle(r, middlePage.getRotation());
+        // remove top line
+        midPageLines.remove(0);
+        // remove top line
+        midPageLines.remove(1);
+        // add string to left line
+        var leftLine = midPageLines.remove(1);
+        coloredLines.add(new ColoredLine(leftLine, color));
+        for (Line2D line : midPageLines) {
+            coloredLines.add(new ColoredLine(line, color));
+        }
+    }
+
+
+    private void addLinesForLastPageOfSection(SemanticNode semanticNode, Color color, Page lastPage, LayoutGrid layoutGrid) {
+
+        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(lastPage.getNumber() - 1).getColoredLines();
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(lastPage), LINE_WIDTH, LINE_WIDTH);
+        var lastPageLines = createLinesFromRectangle(r, lastPage.getRotation());
+        // remove top line
+        lastPageLines.remove(0);
+        // add string to left line
+        var leftLine = lastPageLines.remove(2);
+        coloredLines.add(new ColoredLine(leftLine, color));
+        for (Line2D line : lastPageLines) {
+            coloredLines.add(new ColoredLine(line, color));
+        }
+    }
+
+
+    private void addLinesForFirstPageOfSection(SemanticNode semanticNode, Color color, Page firstPage, LayoutGrid layoutGrid) {
+
+        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(firstPage.getNumber() - 1).getColoredLines();
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(firstPage), LINE_WIDTH, LINE_WIDTH);
+        var firstPageLines = createLinesFromRectangle(r, firstPage.getRotation());
+        // remove bottom line
+        firstPageLines.remove(2);
+        // add string to top line
+        var firstLine = firstPageLines.remove(0);
+        coloredLines.add(new ColoredLine(firstLine, color));
+        for (Line2D line : firstPageLines) {
+            coloredLines.add(new ColoredLine(line, color));
+        }
+    }
+
+
+    private String buildTreeIdString(SemanticNode semanticNode) {
+
+        return semanticNode.getTreeId().stream().map(Object::toString).collect(Collectors.joining("."));
+    }
+
+
+    /*
+    A __________________ B
+     |                  |
+     |                  |
+     |                  |
+     |                  |
+    D|__________________| C
+    The returned List are the lines [AB, BC, DC, AD]
+    The List is reordered, such that the order of the returned lines are always as viewed on the page.
+     */
+    private List<Line2D> createLinesFromRectangle(Rectangle2D r, int pageRotation) {
+        // +0.5 to join the lines
+        List<Line2D> lines = new ArrayList<>(4);
+        float lineWidthCorrection = LINE_WIDTH * 0.5f;
+        Point2D.Float a = new Point2D.Float((float) r.getMinX(), (float) r.getMaxY());
+        Point2D.Float a1 = new Point2D.Float((float) r.getMinX() - lineWidthCorrection, (float) r.getMaxY());
+        Point2D.Float b = new Point2D.Float((float) r.getMaxX(), (float) r.getMaxY());
+        Point2D.Float b1 = new Point2D.Float((float) r.getMaxX() + lineWidthCorrection, (float) r.getMaxY());
+        Point2D.Float c = new Point2D.Float((float) r.getMaxX(), (float) r.getMinY());
+        Point2D.Float c1 = new Point2D.Float((float) r.getMaxX() + lineWidthCorrection, (float) r.getMinY());
+        Point2D.Float d = new Point2D.Float((float) r.getMinX(), (float) r.getMinY());
+        Point2D.Float d1 = new Point2D.Float((float) r.getMinX() - lineWidthCorrection, (float) r.getMinY());
+        lines.add(new Line2D.Float(a1, b1));
+        lines.add(new Line2D.Float(b, c));
+        lines.add(new Line2D.Float(d1, c1));
+        lines.add(new Line2D.Float(a, d));
+
+        return switch (pageRotation) {
+            case 90 -> {
+                Collections.rotate(lines, 1);
+                yield lines;
+            }
+            case 180 -> {
+                Collections.rotate(lines, 2);
+                yield lines;
+            }
+            case 270 -> {
+                Collections.rotate(lines, 3);
+                yield lines;
+            }
+
+            default -> lines;
+        };
+    }
+
+
+    private static boolean isNotSectionOrTableCellOrDocument(SemanticNode semanticNode) {
+
+        return !(semanticNode.getType().equals(NodeType.DOCUMENT) || semanticNode.getType().equals(NodeType.SECTION) || semanticNode.getType().equals(NodeType.TABLE_CELL));
+    }
+
+
+    private void addAsRectangle(SemanticNode semanticNode, LayoutGrid layoutGrid, Color color) {
+
+        semanticNode.getBBox()
+                .forEach((page, textBBox) -> layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredRectangles().add(new ColoredRectangle(textBBox, color)));
+    }
+
+}
@@ -0,0 +1,179 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
+
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Rectangle2D;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.PDResources;
+import org.apache.pdfbox.pdmodel.font.PDFont;
+import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentGroup;
+import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentProperties;
+import org.apache.pdfbox.pdmodel.graphics.state.PDExtendedGraphicsState;
+import org.apache.pdfbox.util.Matrix;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.ColoredLine;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.ColoredRectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.FilledRectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.LayoutGrid;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ViewerDocumentService {
+
+    private static final String layerName = "Layout grid";
+
+    private static final int FONT_SIZE = 10;
+    public static final float LINE_WIDTH = 1f;
+
+    private final LayoutGridService layoutGridService;
+
+
+    @SneakyThrows
+    public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream) {
+
+        log.info("Start Viewer Document Creation");
+        LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
+        // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
+        // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
+        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
+        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate);
+        PDFont font = PDType1Font.HELVETICA;
+
+        for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
+            PDPage pdPage = pdDocument.getPage(pageNumber);
+
+            AffineTransform textDeRotationMatrix = getTextDeRotationTransform(pdPage);
+            addLayerToPageRessources(pdPage);
+
+            // We need to save the graphics state before, such that our appended content cannot be affected by previous content streams with side effects,
+            // e.g. not escaped matrix transformations.
+            escapePreviousContents(pdDocument, pdPage);
+
+            VisualizationsOnPage visualizationsOnPage = layoutGrid.getVisualizationsPerPages().get(pageNumber);
+            assert pageNumber == visualizationsOnPage.getPageNumber();
+            // We need to append to the content stream, otherwise the content could be overlapped by following content.
+            try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true)) {
+
+                contentStream.beginMarkedContent(COSName.OC, layer);
+                contentStream.saveGraphicsState();
+
+                contentStream.setLineWidth(LINE_WIDTH);
+                for (ColoredLine coloredLine : visualizationsOnPage.getColoredLines()) {
+                    contentStream.setStrokingColor(coloredLine.color());
+                    contentStream.moveTo((float) coloredLine.line().getX1(), (float) coloredLine.line().getY1());
+                    contentStream.lineTo((float) coloredLine.line().getX2(), (float) coloredLine.line().getY2());
+                    contentStream.stroke();
+                }
+                for (ColoredRectangle coloredRectangle : visualizationsOnPage.getColoredRectangles()) {
+                    contentStream.setStrokingColor(coloredRectangle.color());
+                    Rectangle2D r = coloredRectangle.rectangle2D();
+                    contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
+                    contentStream.stroke();
+                }
+                for (FilledRectangle filledRectangle : visualizationsOnPage.getFilledRectangles()) {
+                    contentStream.setNonStrokingColor(filledRectangle.color());
+                    PDExtendedGraphicsState graphicsState = new PDExtendedGraphicsState();
+                    graphicsState.setNonStrokingAlphaConstant(filledRectangle.alpha());
+                    contentStream.setGraphicsStateParameters(graphicsState);
+                    Rectangle2D r = filledRectangle.rectangle2D();
+                    contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
+                    contentStream.fill();
+                }
+                for (PlacedText placedText : visualizationsOnPage.getPlacedTexts()) {
+                    contentStream.setFont(font, FONT_SIZE);
+                    contentStream.beginText();
+                    Matrix textMatrix = new Matrix((float) textDeRotationMatrix.getScaleX(),
+                            (float) textDeRotationMatrix.getShearX(),
+                            (float) textDeRotationMatrix.getShearY(),
+                            (float) textDeRotationMatrix.getScaleY(),
+                            (float) placedText.lineStart().getX(),
+                            (float) placedText.lineStart().getY());
+                    textMatrix.translate(-((font.getStringWidth(placedText.text()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
+                    contentStream.setTextMatrix(textMatrix);
+                    contentStream.showText(placedText.text());
+                    contentStream.endText();
+                }
+                contentStream.restoreGraphicsState();
+                contentStream.endMarkedContent();
+            }
+            dictionariesToUpdate.add(pdPage.getCOSObject());
+            dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
+        }
+        dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
+        pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
+        log.info("Saved Viewer Document");
+    }
+
+
+    private static void addLayerToPageRessources(PDPage pdPage) {
+
+        PDResources resources = pdPage.getResources();
+        if (resources == null) {
+            resources = new PDResources();
+            pdPage.setResources(resources);
+        }
+    }
+
+
+    private static void escapePreviousContents(PDDocument pdDocument, PDPage pdPage) throws IOException {
+
+        try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.PREPEND, false)) {
+            contentStream.saveGraphicsState();
+        }
+        try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, false)) {
+            contentStream.restoreGraphicsState();
+        }
+    }
+
+
+    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate) {
+
+        PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
+        PDOptionalContentProperties ocprops = catalog.getOCProperties();
+        if (ocprops == null) {
+            ocprops = new PDOptionalContentProperties();
+            catalog.setOCProperties(ocprops);
+        }
+        PDOptionalContentGroup layer = null;
+        if (ocprops.hasGroup(layerName)) {
+            layer = ocprops.getGroup(layerName);
+        } else {
+            layer = new PDOptionalContentGroup(layerName);
+            ocprops.addGroup(layer);
+        }
+        ocprops.setGroupEnabled(layer, false);
+        dictionariesToUpdate.add(catalog.getCOSObject());
+        return layer;
+    }
+
+
+    private static AffineTransform getTextDeRotationTransform(PDPage page) {
+
+        return AffineTransform.getQuadrantRotateInstance(switch (page.getRotation()) {
+            case 90 -> 3;
+            case 180 -> 2;
+            case 270 -> 1;
+            default -> 0;
+        });
+    }
+
+}
@@ -0,0 +1,55 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import lombok.experimental.UtilityClass;
+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.text.TextPosition;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@UtilityClass
+public class MarkedContentUtils {
+
+    public static final String HEADER = "Header";
+    public static final String FOOTER = "Footer";
+
+    public Rectangle2D getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {
+
+        if (markedContents == null) {
+            return null;
+        }
+
+        var markedContentByYPosition = markedContents.stream()
+                .filter(m -> m.getTag().equals("Artifact"))
+                .filter(m -> m.getProperties() != null)
+                .filter(m -> m.getProperties().getItem("Subtype") != null)
+                .filter(m -> ((COSName) m.getProperties().getItem("Subtype")).getName().equals(subtype))
+                .map(PDMarkedContent::getContents).flatMap(Collection::stream)
+                .filter(t -> t instanceof TextPosition)
+                .map(t -> (TextPosition) t)
+                .filter(t -> !t.getUnicode().equals(" "))
+                .collect(Collectors.groupingBy(TextPosition::getY));
+
+        if (markedContentByYPosition.isEmpty()) {
+            return null;
+        }
+
+        return RectangleTransformations.rectangle2DBBox(markedContentByYPosition.values().stream()
+                .map(textPositions -> new TextPositionSequence(textPositions.stream()
+                        .toList(), 0, true)
+                        .getRectangle())
+                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList()));
+    }
+
+
+    public boolean intersects(TextPageBlock textBlock, Map<String, Rectangle2D> markedContentBboxPerType, String type) {
+        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight());
+    }
+
+}
@@ -12,7 +12,6 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.apache.pdfbox.pdmodel.font.Standard14Fonts;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
@@ -72,7 +71,7 @@ public class PdfVisualisationUtility {

        contentStream.beginText();
        contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
-        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
+        contentStream.setFont(PDType1Font.HELVETICA, 10);
        contentStream.showText(string);
        contentStream.endText();
        contentStream.close();
@@ -152,7 +151,7 @@ public class PdfVisualisationUtility {


    @SneakyThrows
-    public static void drawLine2DList(PDDocument pdDocument, int pageNumber, List<Line2D> line2DS, Options options) {
+    public static void drawLine2DList(PDDocument pdDocument, int pageNumber, List<? extends Line2D> line2DS, Options options) {

        var pdPage = pdDocument.getPage(pageNumber - 1);
        var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true);
@@ -4,39 +4,54 @@ plugins {
    application
    id("com.knecon.fforesight.java-conventions")
    id("org.springframework.boot") version "3.1.2"
-    id("io.spring.dependency-management") version "1.1.0"
-    id("org.sonarqube") version "4.2.1.3168"
-    id("io.freefair.lombok") version "8.1.0"
-//    id("org.graalvm.buildtools.native") version "0.9.23"
+    id("io.spring.dependency-management") version "1.1.3"
+    id("org.sonarqube") version "4.3.0.3225"
+    id("io.freefair.lombok") version "8.2.2"
+    id("org.graalvm.buildtools.native") version "0.9.24"
 }

 description = "layoutparser-service-server"

-val jacksonVersion = "2.15.0-rc2"
-val pdfBoxVersion = "3.0.0-alpha2"
+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0-RC1"
+
+

 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.iqser.red.commons:storage-commons:azNative2")
    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
-    implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
+    implementation("com.amazonaws:aws-java-sdk-s3:1.12.533")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.2")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")

+
+//    implementation("com.azure:azure-storage-blob:12.24.0-beta.1")
+//    implementation("com.azure.spring:azure-spring-cloud-storage:2.14.0")
+
    // for integration testing only
    testImplementation(project(":layoutparser-service-internal-api"))

-    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
-    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
-    testImplementation("com.iqser.red.commons:jackson-commons:1.0.0")
+
+    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.1.2")
+    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.138.0")
+    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
-    testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.2")
+}
+
+
+configurations {
+    all {
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-logging")
+        exclude(group = "commons-logging", module = "commons-logging")
+    }
 }

 /*
@@ -48,6 +63,8 @@ We traced the issue to storage-commons and there to azure blob storage, this is
 https://github.com/Azure/azure-sdk-for-java/issues/36001

 //AOT is springs ahead of time compilation, it is a code analysis which generates new code, therefore checkstyle and pmd can be disabled on them.
+*/
+
 tasks.named<Checkstyle>("checkstyleAot") {
    enabled = false
 }
@@ -60,7 +77,7 @@ tasks.named<Pmd>("pmdAot") {
 tasks.named<Pmd>("pmdAotTest") {
    enabled = false
 }
-*/
+

 sonarqube {
    properties {
@@ -70,6 +87,12 @@ sonarqube {
 }

 tasks.named<BootBuildImage>("bootBuildImage") {
+
+    environment.put("BP_NATIVE_IMAGE_BUILD_ARGUMENTS", "-Djava.security.properties=/workspace/BOOT-INF/classes/custom.security --initialize-at-run-time=org.slf4j,org.apache.log4,org.apache.commons.logging --trace-class-initialization=org.apache.commons.logging.LogFactory")
+
+    environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
+    environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
+
    imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
    if (project.hasProperty("buildbootDockerHostNetwork")) {
        network.set("host")
@@ -0,0 +1,10 @@
+[
+  {
+    "name": "org.apereo.cas.web.CasWebApplicationContext",
+    "allDeclaredConstructors": true
+  }
+]
+
+
+
+
@@ -0,0 +1 @@
+jdk.jar.disabledAlgorithms=MD2, MD5, RSA, DSA
@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server;

+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
 import java.awt.Color;
 import java.io.File;
 import java.io.FileInputStream;
@@ -17,15 +19,17 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
@@ -46,12 +50,27 @@ public class BdrJsonBuildTest extends AbstractTest {

        try (InputStream inputStream = new FileInputStream(filename)) {
            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
-                return layoutParsingPipeline.parseLayoutWithTimer(LayoutParsingType.TAAS, pdDocument, new ImageServiceResponse(), new TableServiceResponse());
+                return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                        pdDocument,
+                        new ImageServiceResponse(),
+                        new TableServiceResponse()));
            }
        }
    }


+    @Test
+    @SneakyThrows
+    public void testBDRFile199865() {
+
+        File file = new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile();
+        Document document = buildGraph(file);
+        ResearchDocumentData researchDocumentData = TaasDocumentDataMapper.fromDocument(document);
+        assertThat(researchDocumentData).isNotNull();
+
+    }
+
+
    @Test
    @Disabled
    public void writeBDRDocumentData() throws IOException {
@@ -30,10 +30,11 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;

 import lombok.AllArgsConstructor;
@@ -94,10 +95,10 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

-        Document documentGraph = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                Loader.loadPDF(pdfFileResource.getInputStream()),
                new ImageServiceResponse(),
-                new TableServiceResponse());
+                new TableServiceResponse()));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -15,6 +15,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;

 import lombok.SneakyThrows;
@@ -29,7 +30,7 @@ public class BuildDocumentGraphTest extends AbstractTest {
    @Disabled
    public void buildMetolachlor() {

-        Document documentGraph = buildGraph("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
+        Document documentGraph = buildGraph("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
        assertEquals(221, documentGraph.getPages().size());
        assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
        assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
@@ -39,6 +40,13 @@ public class BuildDocumentGraphTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(String filename) {

+        return buildGraph(filename, LayoutParsingType.REDACT_MANAGER);
+    }
+
+
+    @SneakyThrows
+    protected Document buildGraph(String filename, LayoutParsingType layoutParsingType) {
+
        if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
            prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json");
        } else {
@@ -47,10 +55,10 @@ public class BuildDocumentGraphTest extends AbstractTest {
        ClassPathResource fileResource = new ClassPathResource(filename);

        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream)) {
-            return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(layoutParsingType,
                    pdDocument,
                    layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
-                    new TableServiceResponse());
+                    new TableServiceResponse()));
        }
    }

@@ -15,9 +15,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;

 import lombok.SneakyThrows;
@@ -54,10 +55,10 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentGraphTest {
    @SneakyThrows
    private void writeJsons(Path filename) {

-        Document documentGraph = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                Loader.loadPDF(new FileInputStream(filename.toFile())),
                new ImageServiceResponse(),
-                new TableServiceResponse());
+                new TableServiceResponse()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.server.graph;
 import java.awt.Color;
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;

 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -15,15 +16,17 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.textbloc
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;

+@Slf4j
 public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {

    @Test
    @SneakyThrows
-    @Disabled
+//    @Disabled
    public void visualizeMetolachlor() {

-        String filename = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
+        String filename = "files/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (4).pdf";
        visualizePdf(filename);
    }

@@ -33,7 +36,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
    @Disabled
    public void visualizeRotatedTestDocument() {

-        String filename = "files/211";
+        String filename = "files/new/RotateTestFile.pdf";
        visualizePdf(filename);
    }

@@ -43,7 +46,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
    @Disabled
    public void visualizeCraftedDocument() {

-        String filename = "files/crafted document";
+        String filename = "files/crafted document.pdf";
        visualizePdf(filename);
    }

@@ -60,15 +63,18 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {

    private static void visualizeSemanticNodes(String filename, Document documentGraph, TextBlock textBlock) throws IOException {

-        File tmpFile = File.createTempFile(filename, "SEMANTIC_NODES_BBOX.pdf");
-        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+        File tmpFile = new File("/tmp/" + Path.of(filename).getFileName().toString() + "_SEMANTIC_NODES_BBOX.pdf");
+        ClassPathResource fileResource = new ClassPathResource(filename);

        try (var fileStream = fileResource.getInputStream();//
             PDDocument pdDocument = Loader.loadPDF(fileStream)//
        ) {
-                PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
-                PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
-                pdDocument.save(tmpFile);
+            log.info("drawing document");
+            PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
+            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
+            log.info("saving document");
+            pdDocument.save(tmpFile);
+            log.info("saved document");
        }
    }

@@ -0,0 +1,34 @@
+package com.knecon.fforesight.service.layoutparser.server.graph;
+
+import java.io.FileOutputStream;
+import java.nio.file.Path;
+
+import org.apache.pdfbox.Loader;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+
+import lombok.SneakyThrows;
+
+public class ViewerDocumentTest extends BuildDocumentGraphTest {
+
+    @Test
+    @SneakyThrows
+    public void testViewerDocument() {
+
+        LayoutGridService layoutGridService = new LayoutGridService();
+        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
+        String fileName = "files/new/VV-511309_OCR.pdf";
+        Document document = buildGraph(fileName, LayoutParsingType.DOCUMINE);
+        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
+        try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getInputStream()); var out = new FileOutputStream(tmpFileName)) {
+            viewerDocumentService.createViewerDocument(pdDocument, document, out);
+        }
+    }
+
+}
@@ -1,46 +1,38 @@
 package com.knecon.fforesight.service.layoutparser.server.segmentation;

-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.awt.geom.Rectangle2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.stream.Collectors;
-
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+import lombok.SneakyThrows;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
-import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+import java.awt.geom.Rectangle2D;
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;

-import lombok.SneakyThrows;
+import static org.assertj.core.api.Assertions.assertThat;

 public class PdfSegmentationServiceTest extends AbstractTest {

    @Autowired
-    private PdfParsingService pdfParsingService;
+    private LayoutParsingPipeline layoutParsingPipeline;

    @Autowired
    private ObjectMapper objectMapper;
@@ -57,12 +49,13 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @Autowired
    private SectionsBuilderService sectionsBuilderService;

+
    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {

-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(LayoutParsingType.REDACT_MANAGER,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
                originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(new TableServiceResponse()),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(new ImageServiceResponse()));
+                new ImageServiceResponse(),
+                new TableServiceResponse());

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -14,7 +14,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.DividingCol
 import com.knecon.fforesight.service.layoutparser.processor.services.GapDetectionService;
 import com.knecon.fforesight.service.layoutparser.processor.services.GapsAcrossLinesService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;
@@ -30,7 +30,7 @@ class GapAcrossLinesDetectionServiceTest {
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_COLUMNS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
-        List<PageInformation> pageInformations = TextPositionSequenceSorter.getSortedTextPositionsWithPages(filename).stream().map(PageInformationService::build).toList();
+        List<PageInformation> pageInformations = PageContentExtractor.getSortedPageContents(filename).stream().map(PageInformationService::build).toList();
        List<List<Rectangle2D>> columnsPerPage = new LinkedList<>();
        System.out.printf("Finished TextPosition Extraction in %d ms%n", System.currentTimeMillis() - start);
        System.out.println("start column detection");
@@ -56,7 +56,7 @@ class GapAcrossLinesDetectionServiceTest {
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_COLUMNS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
-        List<PageContents> sortedTextPositionSequencesPerPage = TextPositionSequenceSorter.getSortedTextPositionsWithPages(filename);
+        List<PageContents> sortedTextPositionSequencesPerPage = PageContentExtractor.getSortedPageContents(filename);
        List<List<Rectangle2D>> columnsPerPage = new LinkedList<>();
        System.out.printf("Finished TextPosition Extraction in %d ms%n", System.currentTimeMillis() - start);
        System.out.println("start column detection");
@@ -13,7 +13,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.PageInformatio
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.InvisibleTableDetectionService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

@@ -28,7 +28,7 @@ class InvisibleTableDetectionServiceTest {

        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TABLE.pdf").toString();
-        List<PageInformation> pageContents = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName).stream().map(PageInformationService::build).collect(Collectors.toList());
+        List<PageInformation> pageContents = PageContentExtractor.getSortedPageContents(fileName).stream().map(PageInformationService::build).collect(Collectors.toList());

        int pageNumber = 1;
        Rectangle2D tableBBox = pageContents.get(0)
@@ -7,7 +7,7 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;

 import lombok.SneakyThrows;

@@ -20,7 +20,7 @@ class MainBodyTextFrameExtractionServiceTest {

        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
        String tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_MAIN_BODY.pdf").toString();
-        List<PageContents> sortedTextPositionSequence = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName);
+        List<PageContents> sortedTextPositionSequence = PageContentExtractor.getSortedPageContents(fileName);


    }
@@ -9,22 +9,21 @@ import org.junit.jupiter.api.Test;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;

-class TextPositionSequenceSorterTest {
+class PageContentExtractorTest {

    @Test
-    @Disabled
    @SneakyThrows
    public void testTextPositionSequenceExtraction() {

-        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
+        String fileName = "files/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (4).pdf";
        var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TEXT_POSITION_SEQUENCES.pdf").toString();

-        List<PageContents> textPositionPerPage = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName);
+        List<PageContents> textPositionPerPage = PageContentExtractor.getSortedPageContents(fileName);

        PdfDraw.drawRectanglesPerPageNumberedByLine(fileName,
                textPositionPerPage.stream()
@@ -8,7 +8,7 @@ import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageInformation;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;
@@ -24,7 +24,7 @@ class PageInformationServiceTest {
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_GAPS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
-        List<PageInformation> pageInformations = TextPositionSequenceSorter.getSortedTextPositionsWithPages(filename).stream().map(PageInformationService::build).toList();
+        List<PageInformation> pageInformations = PageContentExtractor.getSortedPageContents(filename).stream().map(PageInformationService::build).toList();
        System.out.printf("Finished TextPosition Extraction in %d ms%n", System.currentTimeMillis() - start);
        System.out.println("start gap detection");
        start = System.currentTimeMillis();
@@ -47,7 +47,7 @@ class PageInformationServiceTest {
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_GAPS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
-        List<PageInformation> pageInformations = TextPositionSequenceSorter.getSortedTextPositionsWithPages(filename).stream().map(PageInformationService::build).toList();
+        List<PageInformation> pageInformations = PageContentExtractor.getSortedPageContents(filename).stream().map(PageInformationService::build).toList();
        System.out.printf("Finished TextPosition Extraction in %d ms%n", System.currentTimeMillis() - start);
        System.out.println("start gap detection");
        start = System.currentTimeMillis();
@@ -0,0 +1,38 @@
+package com.knecon.fforesight.service.layoutparser.server.services;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
+
+import lombok.SneakyThrows;
+
+public class RulingCleaningServiceTest {
+
+    @Test
+    @Disabled
+    @SneakyThrows
+    public void textRulingExtraction() {
+
+        String fileName = "files/BASF/2013-1110704.pdf";
+        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
+        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents("files/BASF/2013-1110704.pdf");
+        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
+
+        RulingCleaningService rulingCleaningService = new RulingCleaningService();
+        List<CleanRulings> cleanRulingsPerPage = new LinkedList<>();
+        for (PageContents pageContent : pageContents) {
+            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings(), 8, 20));
+        }
+    }
+
+}
@@ -54,6 +54,7 @@ public abstract class AbstractTest {
    protected final static String PAGES_FILE_ID = "pages";
    protected final static String TENANT_ID = "tenant";
    protected final static String SECTION_GRID_ID = "section";
+    protected final static String VIEWER_DOCUMENT_ID = "viewer";
    protected final static String SIMPLIFIED_ID = "simplified";


@@ -70,6 +71,7 @@ public abstract class AbstractTest {
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
                .sectionGridStorageId(SECTION_GRID_ID)
+                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }

@@ -110,6 +112,7 @@ public abstract class AbstractTest {
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
                .sectionGridStorageId(SECTION_GRID_ID)
+                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }

@@ -143,6 +146,7 @@ public abstract class AbstractTest {
                .pageFileStorageId(PAGES_FILE_ID)
                .simplifiedTextStorageId(SIMPLIFIED_ID)
                .sectionGridStorageId(SECTION_GRID_ID)
+                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                .build();
    }

@@ -13,7 +13,6 @@ import java.util.Map;
 import java.util.stream.Collectors;

 import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -36,14 +35,14 @@ public class FileSystemBackedStorageService implements StorageService {

    @SneakyThrows
    @Override
-    public InputStreamResource getObject(String tenantId, String objectId) {
+    public void downloadTo(String tenantId, String objectId, File destinationFile) {

        var res = dataMap.get(objectId);
        if (res == null) {
            throw new StorageObjectDoesNotExist(new RuntimeException());
        }
-        return new InputStreamResource(new FileInputStream(res));

+        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
    }


@@ -14,7 +14,6 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
 import org.apache.pdfbox.util.Matrix;
 import org.springframework.core.io.ClassPathResource;

@@ -24,6 +23,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

@@ -143,7 +143,7 @@ public class PdfDraw {
        } else {
            contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
        }
-        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
+        contentStream.setFont(PDType1Font.HELVETICA, 10);
        contentStream.showText(string);
        contentStream.endText();
        contentStream.close();
@@ -206,6 +206,21 @@ public class PdfDraw {
    }


+    @SneakyThrows
+    public static void drawLinesPerPage(String filename, List<List<Ruling>> linesPerPage, String tmpFileName) {
+
+        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
+             PDDocument pdDocument = Loader.loadPDF(inputStream);//
+             var out = new FileOutputStream(tmpFileName)//
+        ) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                PdfVisualisationUtility.drawLine2DList(pdDocument, pageNumber, linesPerPage.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
+            }
+            pdDocument.save(out);
+        }
+    }
+
+
    @Builder
    @AllArgsConstructor
    @Getter
@@ -0,0 +1,8 @@
+#!/bin/bash
+dir=${PWD##*/}
+gradle assemble
+
+buildNumber=${1:-1}
+
+gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$USER-$buildNumber
+echo "nexus.knecon.com:5001/red/${dir}-server-v1:$USER-$buildNumber"
@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}
Author	SHA1	Message	Date
deiflaender	538d4e55a9	Native Build	2023-08-22 13:39:54 +02:00
deiflaender	0cb8029f0a	RED-7461: Fixed pr findings	2023-08-21 16:57:37 +02:00
deiflaender	b270b9c942	RED-7461: Use marked content to classify headers and footers if available	2023-08-21 16:02:24 +02:00
deiflaender	60615ec5d8	RED-7461: First working iteration of header and footer improvement	2023-08-21 15:31:11 +02:00
Renovate Bot	880914a167	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.36.0 (main) See merge request fforesight/layout-parser!50	2023-08-19 09:17:27 +02:00
Renovate Bot	a80a93d2b0	Update dependency com.iqser.red.commons:storage-commons to v2.36.0	2023-08-19 04:15:48 +00:00
Renovate Bot	0afa7e5b12	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.533 (main) See merge request fforesight/layout-parser!49	2023-08-19 03:14:12 +02:00
Renovate Bot	12516ebf22	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.533	2023-08-18 22:13:37 +00:00
Renovate Bot	0dca90c3fe	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.138.0 (main) See merge request fforesight/layout-parser!48	2023-08-18 21:14:23 +02:00
Renovate Bot	2506c9e091	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.138.0	2023-08-18 16:15:53 +00:00
Timo Bejan	83d39ba3a5	Fixed issue with weird colors	2023-08-18 16:21:45 +03:00
Renovate Bot	c09bb06da6	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.35.0 (main) See merge request fforesight/layout-parser!46	2023-08-18 09:17:56 +02:00
Renovate Bot	1793b1138e	Update dependency com.iqser.red.commons:storage-commons to v2.35.0	2023-08-18 04:17:52 +00:00
Renovate Bot	d30735bc49	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.532 (main) See merge request fforesight/layout-parser!45	2023-08-18 03:12:57 +02:00
Renovate Bot	9356db5373	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.532	2023-08-17 22:15:27 +00:00
Renovate Bot	ee766e7150	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.135.0 (main) See merge request fforesight/layout-parser!44	2023-08-17 21:15:12 +02:00
Renovate Bot	a33bbc9abc	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.135.0	2023-08-17 16:16:19 +00:00
Renovate Bot	5758295fac	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.132.0 (main) See merge request fforesight/layout-parser!43	2023-08-17 12:15:34 +02:00
Renovate Bot	8142d0aa09	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.132.0	2023-08-17 07:17:11 +00:00
Renovate Bot	dc80353a5b	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.34.0 (main) See merge request fforesight/layout-parser!42	2023-08-17 09:16:50 +02:00
Renovate Bot	4d856b04b3	Update dependency com.iqser.red.commons:storage-commons to v2.34.0	2023-08-17 04:16:30 +00:00
Renovate Bot	6ba25ecaa0	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.531 (main) See merge request fforesight/layout-parser!41	2023-08-17 06:16:08 +02:00
Renovate Bot	fcdcaf16e9	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.531	2023-08-16 22:15:33 +00:00
Renovate Bot	086e338f4a	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.131.0 (main) See merge request fforesight/layout-parser!40	2023-08-17 00:15:20 +02:00
Renovate Bot	9dbe73f376	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.131.0	2023-08-16 19:16:16 +00:00
Renovate Bot	aaf4015c95	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.130.0 (main) See merge request fforesight/layout-parser!39	2023-08-16 18:16:44 +02:00
Renovate Bot	2b65ad4b4b	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.130.0	2023-08-16 13:13:52 +00:00
Renovate Bot	c0c75f6a0e	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.33.0 (main) See merge request fforesight/layout-parser!38	2023-08-16 09:12:41 +02:00
Renovate Bot	2f4af6e377	Update dependency com.iqser.red.commons:storage-commons to v2.33.0	2023-08-16 04:13:49 +00:00
Renovate Bot	b9a305bf2d	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.530 (main) See merge request fforesight/layout-parser!37	2023-08-16 03:13:39 +02:00
Renovate Bot	db6b6af4d7	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.530	2023-08-15 22:12:09 +00:00
Renovate Bot	d73addf7ed	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.127.0 (main) See merge request fforesight/layout-parser!36	2023-08-15 18:14:12 +02:00
Renovate Bot	c7978c93c2	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.127.0	2023-08-15 13:14:00 +00:00
Kilian Schüttler	457f7d9c66	Merge branch 'RED-7158' into 'main' RED-7158: fix for all page rotations See merge request fforesight/layout-parser!35	2023-08-15 15:07:04 +02:00
Kilian Schuettler	0387cdd143	RED-7158: fix for all page rotations * also make lines thinner	2023-08-15 14:55:41 +02:00
Kilian Schüttler	5c6898b975	Merge branch 'RED-7158' into 'main' RED-7158: add layoutgrid into new ViewerDocument as optional content See merge request fforesight/layout-parser!34	2023-08-15 13:22:06 +02:00
Kilian Schuettler	b7b273b47d	RED-7158: layout grid * downgraded storage-commons to working version	2023-08-15 13:15:44 +02:00
Kilian Schuettler	9aa9cb2d54	RED-7158: add layoutgrid into new ViewerDocument as optional content * set layer to invisible by default	2023-08-15 13:14:16 +02:00
Renovate Bot	ee6c21638f	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.32.0 (main) See merge request fforesight/layout-parser!33	2023-08-15 09:12:24 +02:00
Renovate Bot	1e4475afdf	Update dependency com.iqser.red.commons:storage-commons to v2.32.0	2023-08-15 04:13:57 +00:00
Renovate Bot	708d274ebc	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.529 (main) See merge request fforesight/layout-parser!32	2023-08-15 03:13:02 +02:00
Renovate Bot	a94faad870	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.529	2023-08-14 22:12:08 +00:00
Kilian Schüttler	d854125867	Merge branch 'RED-7158' into 'main' RED-7158: add layoutgrid into new ViewerDocument as optional content See merge request fforesight/layout-parser!31	2023-08-14 16:14:23 +02:00
Kilian Schuettler	63de8ef82d	RED-7158: add layoutgrid into new ViewerDocument as optional content * downgraded storage-commons	2023-08-14 16:07:11 +02:00
Kilian Schuettler	ea0af08c31	RED-7851: add layoutgrid to new viewer document as optional content	2023-08-14 16:06:23 +02:00
Renovate Bot	810caa0624	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.31.0 (main) See merge request fforesight/layout-parser!30	2023-08-12 09:14:34 +02:00
Renovate Bot	c282372dc8	Update dependency com.iqser.red.commons:storage-commons to v2.31.0	2023-08-12 04:15:18 +00:00
Renovate Bot	055ccd3366	Merge branch 'renovate/main-plugins-(non-major)' into 'main' Update plugin io.freefair.lombok to v8.2.2 (main) See merge request fforesight/layout-parser!29	2023-08-12 06:14:53 +02:00
Renovate Bot	4b4c73fb7b	Update plugin io.freefair.lombok to v8.2.2	2023-08-12 01:13:35 +00:00
Renovate Bot	35b9cfd1c2	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.528 (main) See merge request fforesight/layout-parser!28	2023-08-12 03:13:18 +02:00
Renovate Bot	9a73b952cf	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.528	2023-08-11 22:13:57 +00:00
Renovate Bot	a1c73094f1	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.126.0 (main) See merge request fforesight/layout-parser!27	2023-08-11 18:17:30 +02:00
Renovate Bot	b79d9946a9	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.126.0	2023-08-11 13:13:58 +00:00
Renovate Bot	a9735daa04	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.30.0 (main) See merge request fforesight/layout-parser!26	2023-08-11 15:13:39 +02:00
Renovate Bot	d00491c15e	Update dependency com.iqser.red.commons:storage-commons to v2.30.0	2023-08-11 10:15:55 +00:00
deiflaender	ed48b6a4bf	RED-6725: Fixed wrong file encoding in container, that leads to not working rules on terms with special chars	2023-08-11 11:16:07 +02:00
Renovate Bot	62eade84b9	Merge branch 'renovate/main-plugins-(non-major)' into 'main' Update plugin io.freefair.lombok to v8.2.1 (main) See merge request fforesight/layout-parser!25	2023-08-11 09:13:51 +02:00
Renovate Bot	d6a217fe70	Update plugin io.freefair.lombok to v8.2.1	2023-08-11 04:15:01 +00:00
Renovate Bot	f1e4d0d52b	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.527 (main) See merge request fforesight/layout-parser!24	2023-08-11 06:14:44 +02:00
Renovate Bot	38a1e8b95f	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.527	2023-08-11 01:14:44 +00:00
Renovate Bot	a371558f8c	Merge branch 'renovate/main-plugins-(non-major)' into 'main' Update plugin io.spring.dependency-management to v1.1.3 (main) See merge request fforesight/layout-parser!23	2023-08-10 21:14:45 +02:00
Renovate Bot	b716b187eb	Update plugin io.spring.dependency-management to v1.1.3	2023-08-10 16:14:50 +00:00
Kilian Schuettler	0be6454a7e	add script for pushing custom images	2023-08-10 15:47:52 +02:00
Renovate Bot	5fde631e04	Merge branch 'renovate/main-spring-boot' into 'main' Update spring boot to v3.1.2 (main) See merge request fforesight/layout-parser!20	2023-08-10 12:15:11 +02:00
Renovate Bot	c076c10840	Update spring boot to v3.1.2	2023-08-10 07:14:23 +00:00
Renovate Bot	24104f8cc1	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.29.0 (main) See merge request fforesight/layout-parser!17	2023-08-10 09:14:03 +02:00
Renovate Bot	3632dd4667	Update dependency com.iqser.red.commons:storage-commons to v2.29.0	2023-08-10 04:14:44 +00:00
Renovate Bot	063aa8bfe1	Merge branch 'renovate/main-com.iqser.red.commons-jackson-commons-1.x' into 'main' Update dependency com.iqser.red.commons:jackson-commons to v1.3.0 (main) See merge request fforesight/layout-parser!16	2023-08-10 06:14:25 +02:00
Renovate Bot	d2716a60e9	Update dependency com.iqser.red.commons:jackson-commons to v1.3.0	2023-08-10 01:13:56 +00:00
Renovate Bot	8f08a8c62b	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.526 (main) See merge request fforesight/layout-parser!22	2023-08-10 03:13:38 +02:00
Renovate Bot	091cb73622	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.526	2023-08-09 22:14:12 +00:00
Renovate Bot	d3b0bc430f	Merge branch 'renovate/main-plugins-(non-major)' into 'main' Update Plugins (non-major) (main) See merge request fforesight/layout-parser!15	2023-08-10 00:13:58 +02:00
Renovate Bot	e3c12bc1bb	Update Plugins (non-major)	2023-08-09 19:14:15 +00:00
Renovate Bot	f6f7a0a952	Merge branch 'renovate/main-jacksonversion' into 'main' Update jacksonVersion to v2.15.2 (main) See merge request fforesight/layout-parser!13	2023-08-09 21:13:57 +02:00
Renovate Bot	96df6e3145	Update jacksonVersion to v2.15.2	2023-08-09 16:12:54 +00:00
Renovate Bot	574e5ad425	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.525 (main) See merge request fforesight/layout-parser!12	2023-08-09 18:12:37 +02:00
Renovate Bot	0611e56baa	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.525	2023-08-09 13:13:33 +00:00
Kilian Schüttler	442c1dafea	Merge branch 'update-pdfbox' into 'main' update PDFBox Version See merge request fforesight/layout-parser!19	2023-08-09 12:48:14 +02:00
Kilian Schüttler	33bc532eac	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.124.0 (main) See merge request fforesight/layout-parser!18	2023-08-09 12:42:38 +02:00
Kilian Schuettler	4bd6e7e343	update PDFBox Version	2023-08-09 12:41:28 +02:00
Renovate Bot	159ac6348c	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.124.0	2023-08-09 10:14:27 +00:00
Kilian Schuettler	17259ed805	add renovate, fix checkstyle	2023-08-09 10:11:02 +02:00
Dominique Eifländer	67bf5cbaa8	Merge branch 'RED-6725' into 'main' RED-6725: Install fonts See merge request fforesight/layout-parser!11	2023-08-09 10:07:05 +02:00
deiflaender	f8a3cbc147	RED-6725: Install fonts	2023-08-09 10:00:07 +02:00
Andrei Isvoran	a3d4fbe3a3	Merge branch 'RED-6864' into 'main' RED-6864 - Switch to DELETE_ON_CLOSE See merge request fforesight/layout-parser!10	2023-08-09 08:37:07 +02:00
Andrei Isvoran	5c1dca5933	RED-6864 - Switch to DELETE_ON_CLOSE	2023-08-09 09:30:37 +03:00
Timo Bejan	f56ab8fa49	Merge branch 'RED-6864' into 'main' RED-6864 - Switch to new storage-commons download See merge request fforesight/layout-parser!9	2023-08-08 17:16:40 +02:00
Andrei Isvoran	cfca5376a0	RED-6864 - Switch to new storage-commons download	2023-08-08 17:16:40 +02:00
Kevin Tumma	0633fa04fb	Update file .gitlab-ci.yml	2023-08-08 13:00:40 +02:00
Dominique Eifländer	659a9abaa5	Merge branch 'DM-165' into 'main' DM-165: Fixed numberFormatException on german local machines See merge request fforesight/layout-parser!8	2023-08-07 12:29:37 +02:00
deiflaender	5877aea3f7	DM-165: Fixed numberFormatException on german local machines	2023-08-07 12:12:00 +02:00
deiflaender	f2b92de827	DM-165: Fixed indexOutOfBounds error in TableNodeFactory	2023-08-05 10:20:05 +02:00
Kilian Schuettler	4a5464d6aa	Refactoring to make downstream refactoring easier	2023-08-04 15:16:36 +02:00
Dominique Eifländer	d9a3bbbd30	Merge branch 'RED-5253' into 'main' RED-5253: Ported last documine changes See merge request fforesight/layout-parser!7	2023-08-04 09:59:57 +02:00
deiflaender	150aea55c0	RED-5253: Ported last documine changes	2023-08-04 09:55:35 +02:00
				`@@ -0,0 +1 @@`
				`jdk.jar.disabledAlgorithms=MD2, MD5, RSA, DSA`