Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main'

Update dependency com.iqser.red.commons:storage-commons to v2.39.0 (main) See merge request fforesight/layout-parser!58
Update dependency com.iqser.red.commons:storage-commons to v2.39.0
2023-08-23 09:17:15 +02:00 · 2023-08-23 04:17:21 +00:00 · 2023-08-23 03:13:39 +02:00 · 2023-08-22 22:16:17 +00:00 · 2023-08-22 21:18:37 +02:00 · 2023-08-22 16:17:49 +00:00
35 changed files with 466 additions and 320 deletions
@@ -38,3 +38,7 @@ build/

 ### VS Code ###
 .vscode/
+gradlew.bat
+gradlew
+gradle.properties
+gradle/
@@ -6,17 +6,17 @@ plugins {
 description = "layoutparser-service-processor"

 val jacksonVersion = "2.15.2"
-val pdfBoxVersion = "3.0.0-RC1"
+val pdfBoxVersion = "3.0.0"

 dependencies {
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.130.0") {
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.140.0") {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.33.0")
+    implementation("com.iqser.red.commons:storage-commons:2.39.0")

    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -2,15 +2,19 @@ package com.knecon.fforesight.service.layoutparser.processor;

 import static java.lang.String.format;

+import java.awt.geom.Rectangle2D;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
@@ -99,7 +103,7 @@ public class LayoutParsingPipeline {
            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));

            try (var out = new ByteArrayOutputStream()) {
-                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out);
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out, false);
                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
            }

@@ -162,9 +166,12 @@ public class LayoutParsingPipeline {
                    stripper.getMaxCharHeight());

            ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case REDACT_MANAGER ->
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS ->
+                        taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE ->
+                        docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
            };
            classificationPage.setCleanRulings(cleanRulings);
            classificationPage.setRotation(rotation);
@@ -173,6 +180,9 @@ public class LayoutParsingPipeline {
            classificationPage.setPageWidth(cropbox.getWidth());
            classificationPage.setPageHeight(cropbox.getHeight());

+            // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
+            classificationPage.setMarkedContentBboxPerType(convertMarkedContents(stripper.getMarkedContents()));
+
            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
                classificationPage.setImages(pdfImages.get(pageNumber));
@@ -201,6 +211,13 @@ public class LayoutParsingPipeline {
    }


+    private Map<String, Rectangle2D> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+        Map<String, Rectangle2D> markedContentBboxes = new HashMap<>();
+        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
+        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
+        return markedContentBboxes;
+    }
+
    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {

        if (!classificationPage.isLandscape()) {
@@ -13,7 +13,6 @@ import java.nio.file.StandardOpenOption;

 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;

@@ -49,7 +48,7 @@ public class LayoutParsingStorageService {
                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
                originDocumentInputStream.close();
            }
-            return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
+            return Loader.loadPDF(tempFile);
        }
    }

@@ -1,7 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;

+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
@@ -11,6 +14,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFre
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;

@Data
@RequiredArgsConstructor
@@ -38,4 +42,6 @@ public class ClassificationPage {

    CleanRulings cleanRulings;

+    private Map<String, Rectangle2D> markedContentBboxPerType = new HashMap<>();
+
 }
@@ -73,6 +73,7 @@ public class TextPageBlock extends AbstractPageBlock {

        return sequences.get(0).getPageWidth();
    }
+    

    public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services;

+import java.util.Comparator;
 import java.util.List;

 import org.springframework.stereotype.Service;
@@ -12,73 +13,70 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

@Service
 public class BodyTextFrameService {

-
+    private static final float RULING_HEIGHT_THRESHOLD = 0.15f; // multiplied with page height. Header/Footer Rulings must be within that border of the page.
+    private static final float RULING_WIDTH_THRESHOLD = 0.75f; // multiplied with page width. Header/Footer Rulings must be at least that wide.

    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {

        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
        Rectangle landscapeBodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), true, layoutParsingType);
        for (ClassificationPage page : classificationDocument.getPages()) {
+//            var updatedBodyTextFrame = getBodyTextFrameFromRulings(page, bodyTextFrame, landscapeBodyTextFrame);
            setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
        }
    }


-/*
-    private Rectangle calculateBodyTextFrameByRulings(List<ClassificationPage> pages) {
+    private Rectangle getBodyTextFrameFromRulings(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {

-        Map<ClassificationPage, List<Ruling>> potentialHeaderRulingsPerPage = new HashMap<>();
-        Map<ClassificationPage, List<Ruling>> potentialFooterRulingsPerPage = new HashMap<>();
-
-        for (var page : pages) {
-            potentialHeaderRulingsPerPage.put(page,
-                    page.getCleanRulings()
-                            .getHorizontal()
-                            .stream()
-                            .filter(ruling -> ruling.getY1() > page.getPageHeight() * 0.8)
-                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
-                            .toList());
-            potentialFooterRulingsPerPage.put(page,
-                    page.getCleanRulings()
-                            .getHorizontal()
-                            .stream()
-                            .filter(ruling -> ruling.getY1() < page.getPageHeight() * 0.2)
-                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
-                            .toList());
+        List<Ruling> potentialFooterRulings = getPotentialFooterRulings(page);
+        List<Ruling> potentialHeaderRulings = getPotentialHeaderRulings(page);
+        var x = bodyTextFrame.getTopLeft().getX();
+        var y = bodyTextFrame.getTopLeft().getY();
+        var w = bodyTextFrame.getWidth();
+        var h = bodyTextFrame.getHeight();
+        if (!potentialFooterRulings.isEmpty()) {
+            h = y + h - potentialFooterRulings.get(0).getTop();
+            y = potentialFooterRulings.get(0).getTop();
        }
+        if (!potentialHeaderRulings.isEmpty()) {
+            h = potentialHeaderRulings.get(0).getBottom() - bodyTextFrame.getTopLeft().getY();
+        }
+        return new Rectangle(new Point(x, y), w, h, page.getPageNumber());
+    }

-        Optional<Ruling> headerRuling = potentialHeaderRulingsPerPage.values()
+
+    private List<Ruling> getPotentialFooterRulings(ClassificationPage page) {
+
+        return page.getCleanRulings()
+                .getHorizontal()
                .stream()
-                .flatMap(Collection::stream)
-                .filter(ruling -> potentialHeaderRulingsPerPage.values()
-                        .stream()
-                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
-                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
-                .min(Comparator.comparingDouble(Ruling::getY1));
+                .filter(ruling -> ruling.getY1() < page.getPageHeight() * RULING_HEIGHT_THRESHOLD)
+                .filter(ruling -> ruling.getWidth() > RULING_WIDTH_THRESHOLD * page.getPageWidth())
+                .sorted(Comparator.comparingDouble(Ruling::getTop))
+                .toList();
+    }

-        Optional<Ruling> footerRuling = potentialFooterRulingsPerPage.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .filter(ruling -> potentialHeaderRulingsPerPage.values()
-                        .stream()
-                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
-                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
-                .max(Comparator.comparingDouble(Ruling::getY1));

-        double maxY = headerRuling.isPresent() ? headerRuling.get().y1 : pages.stream().mapToDouble(ClassificationPage::getPageHeight).max().orElse(Double.MAX_VALUE);
-        double minY = footerRuling.map(ruling -> ruling.y1).orElse(0F);
-        double maxX = pages.stream().mapToDouble(ClassificationPage::getPageWidth).max().orElse(Double.MAX_VALUE);
+    private List<Ruling> getPotentialHeaderRulings(ClassificationPage page) {

-        return new Rectangle(new Point((float) maxX, (float) maxY), (float) 0, (float) minY, -1);
+        return page.getCleanRulings()
+                .getHorizontal()
+                .stream()
+                .filter(ruling -> ruling.getY1() > page.getPageHeight() * (1 - RULING_HEIGHT_THRESHOLD))
+                .filter(ruling -> ruling.getWidth() > RULING_WIDTH_THRESHOLD * page.getPageWidth())
+                .sorted(Comparator.comparingDouble(Ruling::getBottom).reversed())
+                .toList();
    }
-*/


    /**
@@ -129,10 +127,10 @@ public class BodyTextFrameService {
     * @param landscape               Calculate for landscape or portrait
     * @return Rectangle of the text frame
     */
-    private Rectangle calculateBodyTextFrame(List<ClassificationPage> pages,
-                                             FloatFrequencyCounter documentFontSizeCounter,
-                                             boolean landscape,
-                                             LayoutParsingType layoutParsingType) {
+    protected Rectangle calculateBodyTextFrame(List<ClassificationPage> pages,
+                                               FloatFrequencyCounter documentFontSizeCounter,
+                                               boolean landscape,
+                                               LayoutParsingType layoutParsingType) {

        float approximateHeaderLineCount;
        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
@@ -157,6 +155,11 @@ public class BodyTextFrameService {
                        continue;
                    }

+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                        continue;
+                    }
+
                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10) || !layoutParsingType.equals(
                            LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount) {
@@ -233,4 +236,4 @@ public class BodyTextFrameService {

    }

-}
+}
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services;

 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
@@ -26,35 +25,34 @@ public class PageContentExtractor {
    public List<PageContents> getSortedPageContents(String filename) throws IOException {

        List<PageContents> textPositionSequencesPerPage = new LinkedList<>();
-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
+        ClassPathResource pdfResource = new ClassPathResource(filename);

-            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile())) {

-                for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {

-                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-                    stripper.setPageNumber(pageNumber);
-                    stripper.setSortByPosition(true);
-                    stripper.setStartPage(pageNumber);
-                    stripper.setEndPage(pageNumber);
-                    stripper.setPdpage(pdPage);
-                    stripper.getText(pdDocument);
+                PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+                PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+                stripper.setPageNumber(pageNumber);
+                stripper.setSortByPosition(true);
+                stripper.setStartPage(pageNumber);
+                stripper.setEndPage(pageNumber);
+                stripper.setPdpage(pdPage);
+                stripper.getText(pdDocument);

-                    Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
-                            .stream()
-                            .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
+                Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
+                        .stream()
+                        .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));

-                    var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());
+                var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());

-                    textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
-                            RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
-                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox()),
-                            stripper.getRulings()));
-                }
+                textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
+                        RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
+                        RectangleTransformations.toRectangle2D(pdPage.getMediaBox()),
+                        stripper.getRulings()));
            }
        }
-
+    
        return textPositionSequencesPerPage;
    }

@@ -5,6 +5,7 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -62,12 +63,16 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
+        ) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
+        ) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -107,4 +112,4 @@ public class DocuMineClassificationService {
        }
    }

-}
+}
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -51,11 +52,13 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -56,9 +57,11 @@ public class TaasClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
            textBlock.setClassification(PageBlockType.HEADER);
-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -92,28 +92,28 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
     */
    LegacyPDFStreamEngine() throws IOException {

-        addOperator(new BeginText());
-        addOperator(new Concatenate());
-        addOperator(new DrawObject()); // special text version
-        addOperator(new EndText());
-        addOperator(new SetGraphicsStateParameters());
-        addOperator(new Save());
-        addOperator(new Restore());
-        addOperator(new NextLine());
-        addOperator(new SetCharSpacing());
-        addOperator(new MoveText());
-        addOperator(new MoveTextSetLeading());
-        addOperator(new SetFontAndSize());
-        addOperator(new ShowText());
-        addOperator(new ShowTextAdjusted());
-        addOperator(new SetTextLeading());
-        addOperator(new SetMatrix());
-        addOperator(new SetTextRenderingMode());
-        addOperator(new SetTextRise());
-        addOperator(new SetWordSpacing());
-        addOperator(new SetTextHorizontalScaling());
-        addOperator(new ShowTextLine());
-        addOperator(new ShowTextLineAndSpace());
+        addOperator(new BeginText(this));
+        addOperator(new Concatenate(this));
+        addOperator(new DrawObject(this)); // special text version
+        addOperator(new EndText(this));
+        addOperator(new SetGraphicsStateParameters(this));
+        addOperator(new Save(this));
+        addOperator(new Restore(this));
+        addOperator(new NextLine(this));
+        addOperator(new SetCharSpacing(this));
+        addOperator(new MoveText(this));
+        addOperator(new MoveTextSetLeading(this));
+        addOperator(new SetFontAndSize(this));
+        addOperator(new ShowText(this));
+        addOperator(new ShowTextAdjusted(this));
+        addOperator(new SetTextLeading(this));
+        addOperator(new SetMatrix(this));
+        addOperator(new SetTextRenderingMode(this));
+        addOperator(new SetTextRise(this));
+        addOperator(new SetWordSpacing(this));
+        addOperator(new SetTextHorizontalScaling(this));
+        addOperator(new ShowTextLine(this));
+        addOperator(new ShowTextLineAndSpace(this));

        // load additional glyph list for Unicode mapping
        String path = "/org/apache/pdfbox/resources/glyphlist/additional.txt";
@@ -1,32 +1,18 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.parsing;

-import java.awt.color.CMMException;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
-import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
-import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
-import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
-import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
+import org.apache.pdfbox.contentstream.operator.color.*;
+import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
+import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
+import org.apache.pdfbox.contentstream.operator.state.*;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSNumber;
@@ -35,14 +21,11 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.text.TextPosition;

-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-
-import lombok.Getter;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.color.CMMException;
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;

@Getter
@Slf4j
@@ -58,6 +41,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private int minCharHeight;
    private int maxCharHeight;

+
    private float path_x;
    private float path_y;

@@ -68,26 +52,31 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    public PDFLinesTextStripper() throws IOException {

        super();
-        this.addOperator(new SetStrokingColorSpace());
-        this.addOperator(new SetNonStrokingColorSpace());
-        this.addOperator(new SetLineDashPattern());
-        this.addOperator(new SetStrokingDeviceGrayColor());
-        this.addOperator(new SetNonStrokingDeviceGrayColor());
-        this.addOperator(new SetFlatness());
-        this.addOperator(new SetLineJoinStyle());
-        this.addOperator(new SetLineCapStyle());
-        this.addOperator(new SetStrokingDeviceCMYKColor());
-        this.addOperator(new SetNonStrokingDeviceCMYKColor());
-        this.addOperator(new SetLineMiterLimit());
-        this.addOperator(new SetStrokingDeviceRGBColor());
-        this.addOperator(new SetNonStrokingDeviceRGBColor());
-        this.addOperator(new SetRenderingIntent());
-        this.addOperator(new SetStrokingColor());
-        this.addOperator(new SetNonStrokingColor());
-        this.addOperator(new SetStrokingColorN());
-        this.addOperator(new SetNonStrokingColorN());
-        this.addOperator(new SetFontAndSize());
-        this.addOperator(new SetLineWidth());
+        this.addOperator(new SetStrokingColorSpace(this));
+        this.addOperator(new SetNonStrokingColorSpace(this));
+        this.addOperator(new SetLineDashPattern(this));
+        this.addOperator(new SetStrokingDeviceGrayColor(this));
+        this.addOperator(new SetNonStrokingDeviceGrayColor(this));
+        this.addOperator(new SetFlatness(this));
+        this.addOperator(new SetLineJoinStyle(this));
+        this.addOperator(new SetLineCapStyle(this));
+        this.addOperator(new SetStrokingDeviceCMYKColor(this));
+        this.addOperator(new SetNonStrokingDeviceCMYKColor(this));
+        this.addOperator(new SetLineMiterLimit(this));
+        this.addOperator(new SetStrokingDeviceRGBColor(this));
+        this.addOperator(new SetNonStrokingDeviceRGBColor(this));
+        this.addOperator(new SetRenderingIntent(this));
+        this.addOperator(new SetStrokingColor(this));
+        this.addOperator(new SetNonStrokingColor(this));
+        this.addOperator(new SetStrokingColorN(this));
+        this.addOperator(new SetNonStrokingColorN(this));
+        this.addOperator(new SetFontAndSize(this));
+        this.addOperator(new SetLineWidth(this));
+
+
+        addOperator(new BeginMarkedContentSequenceWithProperties(this));
+//        addOperator(new BeginMarkedContentSequence(this));
+        addOperator(new EndMarkedContentSequence(this));
    }


@@ -98,6 +87,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

        //move
        switch (operation) {
+
            case OperatorName.MOVE_TO:
                if (arguments.size() == 2) {
                    Point2D.Float pos = transformPosition(floatValue(arguments.get(0)), floatValue(arguments.get(1)));
@@ -219,7 +209,11 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            // This is a quick and dirt hack
            // Happens for file 216.pdf
            log.debug(e.getMessage());
-            return color.getComponents()[0] == 0 && color.getComponents()[1] == 0 && color.getComponents()[2] == 0 && color.getComponents()[1] == 1;
+            var result = true;
+            for (var component : color.getComponents()) {
+                result = result && component == 0;
+            }
+            return result;
        }
    }

@@ -344,3 +338,4 @@ public class PDFLinesTextStripper extends PDFTextStripper {

 }

+
@@ -25,32 +25,26 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.regex.Pattern;

+import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageTree;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.pdmodel.graphics.PDXObject;
 import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
 import org.apache.pdfbox.pdmodel.interactive.pagenavigation.PDThreadBead;
 import org.apache.pdfbox.text.TextPosition;
 import org.apache.pdfbox.text.TextPositionComparator;
-import org.apache.pdfbox.util.QuickSort;
+
+import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;

 /**
 * This is just a copy except i only adjusted lines 594-607 cause this is a bug in Pdfbox.
@@ -63,6 +57,10 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    private static float defaultDropThreshold = 2.5f;
    private static final boolean useCustomQuickSort;

+    @Getter
+    protected final List<PDMarkedContent> markedContents = new ArrayList<>();
+    protected final Deque<PDMarkedContent> currentMarkedContents = new ArrayDeque<>();
+
    private static final Log LOG = LogFactory.getLog(PDFTextStripper.class);

    // enable the ability to set the default indent/drop thresholds
@@ -196,6 +194,45 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    }


+
+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
+    {
+        PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
+        if (this.currentMarkedContents.isEmpty())
+        {
+            this.markedContents.add(markedContent);
+        }
+        else
+        {
+            PDMarkedContent currentMarkedContent =
+                    this.currentMarkedContents.peek();
+            if (currentMarkedContent != null)
+            {
+                currentMarkedContent.addMarkedContent(markedContent);
+            }
+        }
+        this.currentMarkedContents.push(markedContent);
+    }
+
+    @Override
+    public void endMarkedContentSequence()
+    {
+        if (!this.currentMarkedContents.isEmpty())
+        {
+            this.currentMarkedContents.pop();
+        }
+    }
+
+
+    public void xobject(PDXObject xobject)
+    {
+        if (!this.currentMarkedContents.isEmpty())
+        {
+            this.currentMarkedContents.peek().addXObject(xobject);
+        }
+    }
+
+
    /**
     * This will return the text of a document. See writeText. <br>
     * NOTE: The document must not be encrypted when coming into this method.
@@ -877,7 +914,12 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    textList.add(text);
                }
            }
+            if (!this.currentMarkedContents.isEmpty())
+            {
+                this.currentMarkedContents.peek().addText(text);
+            }
        }
+
    }


@@ -2103,6 +2145,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            this.isHangingIndent = true;
        }

+
    }

 }
@@ -16,6 +16,7 @@ import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.PDResources;
 import org.apache.pdfbox.pdmodel.font.PDFont;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
 import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentGroup;
 import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentProperties;
 import org.apache.pdfbox.pdmodel.graphics.state.PDExtendedGraphicsState;
@@ -39,8 +40,8 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class ViewerDocumentService {

-    private static final String layerName = "Layout grid";

+    private static final String LAYER_NAME = "Layout grid";
    private static final int FONT_SIZE = 10;
    public static final float LINE_WIDTH = 1f;

@@ -48,15 +49,15 @@ public class ViewerDocumentService {


    @SneakyThrows
-    public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream) {
+    public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream, boolean layerVisibilityDefaultValue) {

        log.info("Start Viewer Document Creation");
        LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
        // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
        // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
-        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate);
-        PDFont font = PDType1Font.HELVETICA;
+        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
+        PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);

        for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
            PDPage pdPage = pdDocument.getPage(pageNumber);
@@ -119,6 +120,7 @@ public class ViewerDocumentService {
            dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
        }
        dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
+//        dictionariesToUpdate.add(pdDocument.getDocument().getTrailer());
        pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
        log.info("Saved Viewer Document");
    }
@@ -145,7 +147,7 @@ public class ViewerDocumentService {
    }


-    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate) {
+    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate, boolean layerVisibilityDefaultValue) {

        PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
        PDOptionalContentProperties ocprops = catalog.getOCProperties();
@@ -154,13 +156,13 @@ public class ViewerDocumentService {
            catalog.setOCProperties(ocprops);
        }
        PDOptionalContentGroup layer = null;
-        if (ocprops.hasGroup(layerName)) {
-            layer = ocprops.getGroup(layerName);
+        if (ocprops.hasGroup(LAYER_NAME)) {
+            layer = ocprops.getGroup(LAYER_NAME);
        } else {
-            layer = new PDOptionalContentGroup(layerName);
+            layer = new PDOptionalContentGroup(LAYER_NAME);
            ocprops.addGroup(layer);
        }
-        ocprops.setGroupEnabled(layer, false);
+        ocprops.setGroupEnabled(layer, layerVisibilityDefaultValue);
        dictionariesToUpdate.add(catalog.getCOSObject());
        return layer;
    }
@@ -0,0 +1,55 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import lombok.experimental.UtilityClass;
+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.text.TextPosition;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@UtilityClass
+public class MarkedContentUtils {
+
+    public static final String HEADER = "Header";
+    public static final String FOOTER = "Footer";
+
+    public Rectangle2D getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {
+
+        if (markedContents == null) {
+            return null;
+        }
+
+        var markedContentByYPosition = markedContents.stream()
+                .filter(m -> m.getTag().equals("Artifact"))
+                .filter(m -> m.getProperties() != null)
+                .filter(m -> m.getProperties().getItem("Subtype") != null)
+                .filter(m -> ((COSName) m.getProperties().getItem("Subtype")).getName().equals(subtype))
+                .map(PDMarkedContent::getContents).flatMap(Collection::stream)
+                .filter(t -> t instanceof TextPosition)
+                .map(t -> (TextPosition) t)
+                .filter(t -> !t.getUnicode().equals(" "))
+                .collect(Collectors.groupingBy(TextPosition::getY));
+
+        if (markedContentByYPosition.isEmpty()) {
+            return null;
+        }
+
+        return RectangleTransformations.rectangle2DBBox(markedContentByYPosition.values().stream()
+                .map(textPositions -> new TextPositionSequence(textPositions.stream()
+                        .toList(), 0, true)
+                        .getRectangle())
+                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList()));
+    }
+
+
+    public boolean intersects(TextPageBlock textBlock, Map<String, Rectangle2D> markedContentBboxPerType, String type) {
+        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight());
+    }
+
+}
@@ -12,6 +12,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.apache.pdfbox.pdmodel.font.Standard14Fonts;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
@@ -71,7 +72,7 @@ public class PdfVisualisationUtility {

        contentStream.beginText();
        contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
-        contentStream.setFont(PDType1Font.HELVETICA, 10);
+        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
        contentStream.showText(string);
        contentStream.endText();
        contentStream.close();
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 import java.util.List;
 import java.util.stream.Collectors;

-import org.apache.pdfbox.util.QuickSort;

 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
@@ -13,16 +13,16 @@ plugins {
 description = "layoutparser-service-server"

 val jacksonVersion = "2.15.2"
-val pdfBoxVersion = "3.0.0-RC1"
+val pdfBoxVersion = "3.0.0"

 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.commons:storage-commons:2.33.0")
+    implementation("com.iqser.red.commons:storage-commons:2.39.0")
    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
-    implementation("com.amazonaws:aws-java-sdk-s3:1.12.530")
+    implementation("com.amazonaws:aws-java-sdk-s3:1.12.535")
    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.2")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")

@@ -30,7 +30,7 @@ dependencies {
    testImplementation(project(":layoutparser-service-internal-api"))

    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.1.2")
-    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.130.0")
+    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.140.0")
    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
@@ -1,11 +1,11 @@
 package com.knecon.fforesight.service.layoutparser.server;

+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
 import java.awt.Color;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -17,6 +17,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
@@ -43,19 +44,29 @@ public class BdrJsonBuildTest extends AbstractTest {


    @SneakyThrows
-    protected Document buildGraph(File filename) {
+    protected Document buildGraph(File file) {

-        try (InputStream inputStream = new FileInputStream(filename)) {
-            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
-                return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
-                        pdDocument,
-                        new ImageServiceResponse(),
-                        new TableServiceResponse()));
-            }
+        try (PDDocument pdDocument = Loader.loadPDF(file)) {
+            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                    pdDocument,
+                    new ImageServiceResponse(),
+                    new TableServiceResponse()));
        }
    }


+    @Test
+    @SneakyThrows
+    public void testBDRFile199865() {
+
+        File file = new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile();
+        Document document = buildGraph(file);
+        ResearchDocumentData researchDocumentData = TaasDocumentDataMapper.fromDocument(document);
+        assertThat(researchDocumentData).isNotNull();
+
+    }
+
+
    @Test
    @Disabled
    public void writeBDRDocumentData() throws IOException {
@@ -99,10 +110,7 @@ public class BdrJsonBuildTest extends AbstractTest {

    private static void visualizeSemanticNodes(File file, File resultingFileName, Document document, TextBlock textBlock) throws IOException {

-        try (var fileStream = new FileInputStream(file);//
-             PDDocument pdDocument = Loader.loadPDF(fileStream);//
-             var outputStream = new FileOutputStream(resultingFileName)//
-        ) {
+        try (PDDocument pdDocument = Loader.loadPDF(file); var outputStream = new FileOutputStream(resultingFileName)) {
            PdfDraw.drawDocumentGraph(pdDocument, document);
            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
            pdDocument.save(outputStream);
@@ -96,7 +96,7 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(pdfFileResource.getInputStream()),
+                Loader.loadPDF(pdfFileResource.getFile()),
                new ImageServiceResponse(),
                new TableServiceResponse()));

@@ -12,10 +12,11 @@ import org.springframework.core.io.ClassPathResource;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;

 import lombok.SneakyThrows;

-public class DocumentDataTests extends BuildDocumentGraphTest{
+public class DocumentDataTests extends BuildDocumentTest {
    @Test
    @SneakyThrows
    public void createDocumentDataForAllFiles() {
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -20,10 +19,11 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;

 import lombok.SneakyThrows;

-public class DocumentGraphJsonWritingTest extends BuildDocumentGraphTest {
+public class DocumentGraphJsonWritingTest extends BuildDocumentTest {

    @Test
    @SneakyThrows
@@ -56,7 +56,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentGraphTest {
    private void writeJsons(Path filename) {

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(new FileInputStream(filename.toFile())),
+                Loader.loadPDF(filename.toFile()),
                new ImageServiceResponse(),
                new TableServiceResponse()));

@@ -16,11 +16,12 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Ta
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentGraphMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.tenantcommons.TenantContext;

 import lombok.SneakyThrows;

-public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
+public class DocumentGraphMappingTest extends BuildDocumentTest {

    @Test
    @SneakyThrows
@@ -13,20 +13,21 @@ import org.springframework.core.io.ClassPathResource;

 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
-public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
+public class DocumentGraphVisualizationTest extends BuildDocumentTest {

    @Test
    @SneakyThrows
 //    @Disabled
    public void visualizeMetolachlor() {

-        String filename = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
+        String filename = "files/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (4).pdf";
        visualizePdf(filename);
    }

@@ -66,9 +67,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
        File tmpFile = new File("/tmp/" + Path.of(filename).getFileName().toString() + "_SEMANTIC_NODES_BBOX.pdf");
        ClassPathResource fileResource = new ClassPathResource(filename);

-        try (var fileStream = fileResource.getInputStream();//
-             PDDocument pdDocument = Loader.loadPDF(fileStream)//
-        ) {
+        try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) {
            log.info("drawing document");
            PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
@@ -4,6 +4,7 @@ import java.io.FileOutputStream;
 import java.nio.file.Path;

 import org.apache.pdfbox.Loader;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -11,22 +12,24 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;

 import lombok.SneakyThrows;

-public class ViewerDocumentTest extends BuildDocumentGraphTest {
+public class ViewerDocumentTest extends BuildDocumentTest {

    @Test
+    @Disabled
    @SneakyThrows
    public void testViewerDocument() {

        LayoutGridService layoutGridService = new LayoutGridService();
        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
-        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
-        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
+        String fileName = "files/new/VV-511309_OCR.pdf";
+        Document document = buildGraph(fileName, LayoutParsingType.DOCUMINE);
        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
-        try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getInputStream()); var out = new FileOutputStream(tmpFileName)) {
-            viewerDocumentService.createViewerDocument(pdDocument, document, out);
+        try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getFile()); var out = new FileOutputStream(tmpFileName)) {
+            viewerDocumentService.createViewerDocument(pdDocument, document, out, true);
        }
    }

@@ -1,24 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.server.segmentation;

-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.awt.geom.Rectangle2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -34,8 +15,19 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
-
 import lombok.SneakyThrows;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import java.awt.geom.Rectangle2D;
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;

 public class PdfSegmentationServiceTest extends AbstractTest {

@@ -60,7 +52,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {

-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
                originDocument,
                new ImageServiceResponse(),
                new TableServiceResponse());
@@ -100,7 +92,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(table.getColCount()).isEqualTo(6);
@@ -114,7 +106,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -132,7 +124,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(9);
@@ -150,7 +142,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
@@ -168,7 +160,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

@@ -185,7 +177,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -226,7 +218,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

@@ -243,7 +235,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -258,7 +250,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izZRMS (CZ) fRR Part B7_Page123.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 6);

@@ -277,7 +269,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04_Page11.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 3);

@@ -293,7 +285,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10_Page532.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 9, 0, 0);
@@ -306,7 +298,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -320,7 +312,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
        validateTable(document, 0, 9, 6, 7, 0);
@@ -333,7 +325,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);
        validateTable(document, 0, 10, 6, 0, 1);
@@ -346,7 +338,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);
        validateTable(document, 0, 2, 2, 0, 0);
@@ -361,7 +353,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -377,7 +369,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -392,7 +384,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -407,7 +399,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -422,7 +414,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 4);

@@ -439,7 +431,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -454,7 +446,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource(
                "files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 2);

@@ -469,7 +461,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -483,7 +475,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");

-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));

        validateTableSize(document, 1);

@@ -0,0 +1,31 @@
+package com.knecon.fforesight.service.layoutparser.server.services;
+
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
+
+import lombok.SneakyThrows;
+
+class BodyTextFrameServiceTest extends BuildDocumentTest {
+
+    @Test
+    @SneakyThrows
+    public void testCalculateBodyTextFrame() {
+
+        String filename = "files/211.pdf";
+        String outputFilename = "/tmp/" + Path.of(filename).getFileName() + "_MAINBODY.pdf";
+        ClassificationDocument document = parseLayout(filename, LayoutParsingType.TAAS);
+        PdfDraw.drawRectanglesPerPage(filename,
+                document.getPages().stream().map(page -> List.of(RectangleTransformations.toRectangle2D(page.getBodyTextFrame()))).toList(),
+                outputFilename);
+
+    }
+
+}
@@ -17,11 +17,10 @@ import lombok.SneakyThrows;
 class PageContentExtractorTest {

    @Test
-    @Disabled
    @SneakyThrows
    public void testTextPositionSequenceExtraction() {

-        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
+        String fileName = "files/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (4).pdf";
        var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TEXT_POSITION_SEQUENCES.pdf").toString();

        List<PageContents> textPositionPerPage = PageContentExtractor.getSortedPageContents(fileName);
@@ -5,7 +5,6 @@ import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;

-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
@@ -19,13 +18,13 @@ import lombok.SneakyThrows;
 public class RulingCleaningServiceTest {

    @Test
-    @Disabled
+//    @Disabled
    @SneakyThrows
    public void textRulingExtraction() {

-        String fileName = "files/BASF/2013-1110704.pdf";
+        String fileName = "files/211.pdf";
        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
-        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents("files/BASF/2013-1110704.pdf");
+        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);

        RulingCleaningService rulingCleaningService = new RulingCleaningService();
@@ -1,39 +1,35 @@
-package com.knecon.fforesight.service.layoutparser.server.graph;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
+package com.knecon.fforesight.service.layoutparser.server.utils;

 import java.io.InputStream;

 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;

 import lombok.SneakyThrows;

-public class BuildDocumentGraphTest extends AbstractTest {
+public abstract class BuildDocumentTest extends AbstractTest {

    @Autowired
    protected LayoutParsingPipeline layoutParsingPipeline;


-    @Test
-    @Disabled
-    public void buildMetolachlor() {
+    @SneakyThrows
+    protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {

-        Document documentGraph = buildGraph("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
-        assertEquals(221, documentGraph.getPages().size());
-        assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
-        assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
+        ClassPathResource fileResource = new ClassPathResource(filename);
+        prepareStorage(filename);
+        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream.readAllBytes())) {
+            return layoutParsingPipeline.parseLayout(layoutParsingType, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse());
+        }
    }


@@ -52,14 +48,9 @@ public class BuildDocumentGraphTest extends AbstractTest {
        } else {
            prepareStorage(filename);
        }
-        ClassPathResource fileResource = new ClassPathResource(filename);

-        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream)) {
-            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(layoutParsingType,
-                    pdDocument,
-                    layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
-                    new TableServiceResponse()));
-        }
+        return DocumentGraphFactory.buildDocumentGraph(parseLayout(filename, layoutParsingType));
    }

 }
+
@@ -5,7 +5,6 @@ import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.List;
 import java.util.Map;

@@ -14,6 +13,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
 import org.apache.pdfbox.util.Matrix;
 import org.springframework.core.io.ClassPathResource;

@@ -40,10 +40,8 @@ public class PdfDraw {

    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) throws IOException {

-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
-             PDDocument pdDocument = Loader.loadPDF(inputStream);//
-             var out = new FileOutputStream(tmpFileName)//
-        ) {
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
                        pageNumber,
@@ -58,11 +56,8 @@ public class PdfDraw {

    public static void drawRectanglesPerPageNumberedByLine(String filename, List<List<List<Rectangle2D>>> rectanglesPerPage, String tmpFileName) throws IOException {

-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
-             PDDocument pdDocument = Loader.loadPDF(inputStream);//
-             var out = new FileOutputStream(tmpFileName)//
-        ) {
-
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
                var rectanglesOnPage = rectanglesPerPage.get(pageNumber - 1);
                for (int lineNumber = 0; lineNumber < rectanglesOnPage.size(); lineNumber++) {
@@ -74,13 +69,9 @@ public class PdfDraw {
                            new Point2D.Double(rectanglesInLine.get(0).getX() - (5 + (5 * countNumberOfDigits(lineNumber))), y + 2),
                            pageNumber,
                            PdfVisualisationUtility.Options.builder().stroke(true).build());
-
                }
-
            }
-
            pdDocument.save(out);
-
        }

    }
@@ -143,7 +134,7 @@ public class PdfDraw {
        } else {
            contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
        }
-        contentStream.setFont(PDType1Font.HELVETICA, 10);
+        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
        contentStream.showText(string);
        contentStream.endText();
        contentStream.close();
@@ -184,11 +175,8 @@ public class PdfDraw {
    @SneakyThrows
    public static void drawRectanglesAndLinesPerPage(String filename, List<List<Rectangle2D>> list, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) {

-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
-             PDDocument pdDocument = Loader.loadPDF(inputStream);//
-             var out = new FileOutputStream(tmpFileName)//
-        ) {
-
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
 //                PdfVisualisationUtility.drawLine2DList(pdDocument,
 //                        pageNumber,
@@ -201,7 +189,6 @@ public class PdfDraw {
                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().stroke(true).build());
            }
            pdDocument.save(out);
-
        }
    }

@@ -209,12 +196,13 @@ public class PdfDraw {
    @SneakyThrows
    public static void drawLinesPerPage(String filename, List<List<Ruling>> linesPerPage, String tmpFileName) {

-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
-             PDDocument pdDocument = Loader.loadPDF(inputStream);//
-             var out = new FileOutputStream(tmpFileName)//
-        ) {
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                PdfVisualisationUtility.drawLine2DList(pdDocument, pageNumber, linesPerPage.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
+                PdfVisualisationUtility.drawLine2DList(pdDocument,
+                        pageNumber,
+                        linesPerPage.get(pageNumber - 1),
+                        PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
            }
            pdDocument.save(out);
        }
Author	SHA1	Message	Date
Renovate Bot	f89243472c	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.39.0 (main) See merge request fforesight/layout-parser!58	2023-08-23 09:17:15 +02:00
Renovate Bot	ad3612acd4	Update dependency com.iqser.red.commons:storage-commons to v2.39.0	2023-08-23 04:17:21 +00:00
Renovate Bot	630eee6bd7	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.535 (main) See merge request fforesight/layout-parser!57	2023-08-23 03:13:39 +02:00
Renovate Bot	a951911ec8	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.535	2023-08-22 22:16:17 +00:00
Renovate Bot	75e6b88705	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.140.0 (main) See merge request fforesight/layout-parser!56	2023-08-22 21:18:37 +02:00
Renovate Bot	2e0adbdd9a	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.140.0	2023-08-22 16:17:49 +00:00
Renovate Bot	b747742558	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.38.0 (main) See merge request fforesight/layout-parser!55	2023-08-22 15:17:43 +02:00
Renovate Bot	192c9976c1	Update dependency com.iqser.red.commons:storage-commons to v2.38.0	2023-08-22 10:17:05 +00:00
Dominique Eifländer	b251697492	Merge branch 'PDFBox-update' into 'main' upgrade PDFBox to 3.0.0 See merge request fforesight/layout-parser!52	2023-08-22 09:39:53 +02:00
Renovate Bot	22d6b25fe4	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.534 (main) See merge request fforesight/layout-parser!53	2023-08-22 09:17:26 +02:00
Renovate Bot	e6bcd6fb2b	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.534	2023-08-22 04:17:20 +00:00
Renovate Bot	2847adde22	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.37.0 (main) See merge request fforesight/layout-parser!54	2023-08-22 06:16:54 +02:00
Renovate Bot	7cf67d7121	Update dependency com.iqser.red.commons:storage-commons to v2.37.0	2023-08-22 01:17:23 +00:00
Kilian Schuettler	3a18923ef5	upgrade PDFBox to 3.0.0 * disable experimental ruling header stuff	2023-08-21 17:54:20 +02:00
Kilian Schuettler	2b15fd1d3c	RED-7461: improve header/footer recognition	2023-08-21 17:49:13 +02:00
Dominique Eifländer	3722fff476	Merge branch 'RED-7461' into 'main' Red 7461 See merge request fforesight/layout-parser!51	2023-08-21 17:08:10 +02:00
deiflaender	0cb8029f0a	RED-7461: Fixed pr findings	2023-08-21 16:57:37 +02:00
deiflaender	b270b9c942	RED-7461: Use marked content to classify headers and footers if available	2023-08-21 16:02:24 +02:00
deiflaender	60615ec5d8	RED-7461: First working iteration of header and footer improvement	2023-08-21 15:31:11 +02:00
Renovate Bot	880914a167	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.36.0 (main) See merge request fforesight/layout-parser!50	2023-08-19 09:17:27 +02:00
Renovate Bot	a80a93d2b0	Update dependency com.iqser.red.commons:storage-commons to v2.36.0	2023-08-19 04:15:48 +00:00
Renovate Bot	0afa7e5b12	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.533 (main) See merge request fforesight/layout-parser!49	2023-08-19 03:14:12 +02:00
Renovate Bot	12516ebf22	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.533	2023-08-18 22:13:37 +00:00
Renovate Bot	0dca90c3fe	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.138.0 (main) See merge request fforesight/layout-parser!48	2023-08-18 21:14:23 +02:00
Renovate Bot	2506c9e091	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.138.0	2023-08-18 16:15:53 +00:00
Timo Bejan	83d39ba3a5	Fixed issue with weird colors	2023-08-18 16:21:45 +03:00
Renovate Bot	c09bb06da6	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.35.0 (main) See merge request fforesight/layout-parser!46	2023-08-18 09:17:56 +02:00
Renovate Bot	1793b1138e	Update dependency com.iqser.red.commons:storage-commons to v2.35.0	2023-08-18 04:17:52 +00:00
Renovate Bot	d30735bc49	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.532 (main) See merge request fforesight/layout-parser!45	2023-08-18 03:12:57 +02:00
Renovate Bot	9356db5373	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.532	2023-08-17 22:15:27 +00:00
Renovate Bot	ee766e7150	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.135.0 (main) See merge request fforesight/layout-parser!44	2023-08-17 21:15:12 +02:00
Renovate Bot	a33bbc9abc	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.135.0	2023-08-17 16:16:19 +00:00
Renovate Bot	5758295fac	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.132.0 (main) See merge request fforesight/layout-parser!43	2023-08-17 12:15:34 +02:00
Renovate Bot	8142d0aa09	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.132.0	2023-08-17 07:17:11 +00:00
Renovate Bot	dc80353a5b	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.34.0 (main) See merge request fforesight/layout-parser!42	2023-08-17 09:16:50 +02:00
Renovate Bot	4d856b04b3	Update dependency com.iqser.red.commons:storage-commons to v2.34.0	2023-08-17 04:16:30 +00:00
Renovate Bot	6ba25ecaa0	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.531 (main) See merge request fforesight/layout-parser!41	2023-08-17 06:16:08 +02:00
Renovate Bot	fcdcaf16e9	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.531	2023-08-16 22:15:33 +00:00
Renovate Bot	086e338f4a	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.131.0 (main) See merge request fforesight/layout-parser!40	2023-08-17 00:15:20 +02:00
Renovate Bot	9dbe73f376	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.131.0	2023-08-16 19:16:16 +00:00