hotfix: test to add fonts

Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main'
Update dependency com.iqser.red.commons:storage-commons to v2.29.0 (main) See merge request fforesight/layout-parser!17
2023-08-10 10:55:25 +02:00 · 2023-08-10 09:14:03 +02:00 · 2023-08-10 04:14:44 +00:00 · 2023-08-10 06:14:25 +02:00 · 2023-08-10 01:13:56 +00:00 · 2023-08-10 03:13:38 +02:00
40 changed files with 486 additions and 319 deletions
@@ -1,4 +1,21 @@
 include:
  - project: 'gitlab/gitlab'
    ref: 'main'
-    file: 'ci-templates/gradle_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG
@@ -4,21 +4,21 @@ import java.util.Map;
 import java.util.Optional;

 import lombok.Builder;
-
+import lombok.NonNull;

@Builder
 public record LayoutParsingRequest(
-        LayoutParsingType layoutParsingType,
+        @NonNull LayoutParsingType layoutParsingType,
        Map<String, String> identifier,
-        String originFileStorageId,
+        @NonNull String originFileStorageId,
        Optional<String> tablesFileStorageId,
        Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
+        @NonNull String structureFileStorageId,
        String researchDocumentStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId,
-        String simplifiedTextStorageId,
-        String sectionGridStorageId) {
+        @NonNull String textBlockFileStorageId,
+        @NonNull String positionBlockFileStorageId,
+        @NonNull String pageFileStorageId,
+        @NonNull String simplifiedTextStorageId,
+        @NonNull String sectionGridStorageId) {

 }
@@ -5,19 +5,22 @@ plugins {

 description = "layoutparser-service-processor"

+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0-RC1"
+
 dependencies {
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0") {
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.124.0") {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.iqser.red.commons:storage-commons:2.29.0")

-    implementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
-    implementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
-    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
-    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
+    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
 }
@@ -3,32 +3,50 @@ package com.knecon.fforesight.service.layoutparser.processor;
 import static java.lang.String.format;

 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;

 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
+import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;

 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@@ -39,14 +57,18 @@ public class LayoutParsingPipeline {
    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
    private final CvTableParsingAdapter cvTableParsingAdapter;
    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final PdfParsingService pdfParsingService;
    private final SectionsBuilderService sectionsBuilderService;
    private final SectionGridCreatorService sectionGridCreatorService;
    private final TaasClassificationService taasClassificationService;
    private final RedactManagerClassificationService redactManagerClassificationService;
    private final DocuMineClassificationService docuMineClassificationService;
    private final SimplifiedSectionTextService simplifiedSectionTextService;
-
+    private final BodyTextFrameService bodyTextFrameService;
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;

    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {

@@ -63,7 +85,9 @@ public class LayoutParsingPipeline {
                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
            }

-            Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+
            int numberOfPages = originDocument.getNumberOfPages();

            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
@@ -88,15 +112,72 @@ public class LayoutParsingPipeline {
    }


-    public Document parseLayout(LayoutParsingType layoutParsingType,
+    @SneakyThrows
+    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
                                PDDocument originDocument,
                                ImageServiceResponse imageServiceResponse,
                                TableServiceResponse tableServiceResponse) {

-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
-                originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
+        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
+        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
+
+        ClassificationDocument classificationDocument = new ClassificationDocument();
+        List<ClassificationPage> classificationPages = new ArrayList<>();
+
+        originDocument.setAllSecurityToBeRemoved(true);
+        long pageCount = originDocument.getNumberOfPages();
+
+        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+            classificationDocument.setPages(classificationPages);
+            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+            PDPage pdPage = originDocument.getPage(pageNumber - 1);
+            stripper.setPageNumber(pageNumber);
+            stripper.setStartPage(pageNumber);
+            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE)) {
+                stripper.setSortByPosition(true);
+            }
+            stripper.getText(originDocument);
+
+            PDRectangle pdr = pdPage.getMediaBox();
+
+            int rotation = pdPage.getRotation();
+            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+            PDRectangle cropbox = pdPage.getCropBox();
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                    stripper.getRulings(),
+                    stripper.getMinCharWidth(),
+                    stripper.getMaxCharHeight());
+
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            };
+            classificationPage.setCleanRulings(cleanRulings);
+            classificationPage.setRotation(rotation);
+            classificationPage.setLandscape(isLandscape);
+            classificationPage.setPageNumber(pageNumber);
+            classificationPage.setPageWidth(cropbox.getWidth());
+            classificationPage.setPageHeight(cropbox.getHeight());
+
+            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+                classificationPage.setImages(pdfImages.get(pageNumber));
+                imageServiceResponseAdapter.findOcr(classificationPage);
+            }
+
+            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            buildPageStatistics(classificationPage);
+            increaseDocumentStatistics(classificationPage, classificationDocument);
+
+            classificationPages.add(classificationPage);
+        }
+
+
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);

        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
@@ -107,40 +188,39 @@ public class LayoutParsingPipeline {
        sectionsBuilderService.buildSections(classificationDocument);
        sectionsBuilderService.addImagesToSections(classificationDocument);

-        return DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+        return classificationDocument;
    }

+    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {

-    public Document parseLayoutWithTimer(LayoutParsingType layoutParsingType,
-                                         PDDocument originDocument,
-                                         ImageServiceResponse imageServiceResponse,
-                                         TableServiceResponse tableServiceResponse) {
-
-        long start = System.currentTimeMillis();
-
-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
-                originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
-
-        System.out.printf("parsed %d ms", System.currentTimeMillis() - start);
-
-        start = System.currentTimeMillis();
-        switch (layoutParsingType) {
-            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
-            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
+        if (!classificationPage.isLandscape()) {
+            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
        }
-        System.out.printf(", classified %d ms", System.currentTimeMillis() - start);
-
-        start = System.currentTimeMillis();
-        sectionsBuilderService.buildSections(classificationDocument);
-        System.out.printf(", sections built %d ms", System.currentTimeMillis() - start);
-
-        start = System.currentTimeMillis();
-        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-        System.out.printf(", graph constructed %d ms", System.currentTimeMillis() - start);
-        return document;
+        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
+        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
+        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
    }

+
+    private void buildPageStatistics(ClassificationPage classificationPage) {
+
+        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
+        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
+                    continue;
+                }
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
+                    classificationPage.getFontCounter().add(word.getFont());
+                    classificationPage.getFontSizeCounter().add(word.getFontSize());
+                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
+                }
+            }
+        }
+
+    }
+
+
+
 }
@@ -5,6 +5,9 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;

 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
@@ -16,14 +19,15 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@@ -37,10 +41,11 @@ public class LayoutParsingStorageService {

    public PDDocument getOriginFile(String storageId) throws IOException {

-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var originDocumentInputStream = getObject(storageId)) {
            File tempFile = createTempFile("document", ".pdf");
            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
            }
            return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
        }
@@ -49,19 +54,22 @@ public class LayoutParsingStorageService {

    public ImageServiceResponse getImagesFile(String storageId) throws IOException {

-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (InputStream inputStream = getObject(storageId)) {

-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
        }
    }


    public TableServiceResponse getTablesFile(String storageId) throws IOException {

-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+        try (var tableClassificationStream = getObject(storageId)) {

+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
        }
    }

@@ -117,4 +125,13 @@ public class LayoutParsingStorageService {
        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
    }

+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
 }
@@ -5,6 +5,7 @@ import java.util.List;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;

 import lombok.Data;
@@ -35,4 +36,6 @@ public class ClassificationPage {
    private float pageWidth;
    private float pageHeight;

+    CleanRulings cleanRulings;
+
 }
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;

+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

 import lombok.AllArgsConstructor;
@@ -17,5 +18,5 @@ public class PageContents {
    List<TextPositionSequence> sortedTextPositionSequences;
    Rectangle2D cropBox;
    Rectangle2D mediaBox;
-
+    List<Ruling> rulings;
 }
@@ -28,6 +28,7 @@ public class Ruling extends Line2D.Float {
        super(p1, p2);
    }

+
    public Ruling straightenVertical() {

        double y1 = Math.min(getY1(), getY2());
@@ -36,6 +37,7 @@ public class Ruling extends Line2D.Float {
        return new Ruling(new Point2D.Double(x, y1), new Point2D.Double(x, y2));
    }

+
    public Ruling straightenHorizontal() {

        double x1 = Math.min(getX1(), getX2());
@@ -444,6 +446,16 @@ public class Ruling extends Line2D.Float {
    }


+    public boolean almostMatches(Ruling ruling) {
+
+        final float TOLERANCE = 1;
+        return Math.abs(ruling.getX1() - x1) < TOLERANCE &&//
+                Math.abs(ruling.getY1() - y1) < TOLERANCE &&//
+                Math.abs(ruling.getX2() - x2) < TOLERANCE &&//
+                Math.abs(ruling.getY2() - y2) < TOLERANCE;
+    }
+
+
    private enum SOType {
        VERTICAL,
        HRIGHT,
@@ -8,6 +8,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -20,6 +21,65 @@ public class BodyTextFrameService {



+    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {
+
+        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
+        Rectangle landscapeBodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), true, layoutParsingType);
+        for (ClassificationPage page : classificationDocument.getPages()) {
+            setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
+        }
+    }
+
+
+/*
+    private Rectangle calculateBodyTextFrameByRulings(List<ClassificationPage> pages) {
+
+        Map<ClassificationPage, List<Ruling>> potentialHeaderRulingsPerPage = new HashMap<>();
+        Map<ClassificationPage, List<Ruling>> potentialFooterRulingsPerPage = new HashMap<>();
+
+        for (var page : pages) {
+            potentialHeaderRulingsPerPage.put(page,
+                    page.getCleanRulings()
+                            .getHorizontal()
+                            .stream()
+                            .filter(ruling -> ruling.getY1() > page.getPageHeight() * 0.8)
+                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
+                            .toList());
+            potentialFooterRulingsPerPage.put(page,
+                    page.getCleanRulings()
+                            .getHorizontal()
+                            .stream()
+                            .filter(ruling -> ruling.getY1() < page.getPageHeight() * 0.2)
+                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
+                            .toList());
+        }
+
+        Optional<Ruling> headerRuling = potentialHeaderRulingsPerPage.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(ruling -> potentialHeaderRulingsPerPage.values()
+                        .stream()
+                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
+                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
+                .min(Comparator.comparingDouble(Ruling::getY1));
+
+        Optional<Ruling> footerRuling = potentialFooterRulingsPerPage.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(ruling -> potentialHeaderRulingsPerPage.values()
+                        .stream()
+                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
+                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
+                .max(Comparator.comparingDouble(Ruling::getY1));
+
+        double maxY = headerRuling.isPresent() ? headerRuling.get().y1 : pages.stream().mapToDouble(ClassificationPage::getPageHeight).max().orElse(Double.MAX_VALUE);
+        double minY = footerRuling.map(ruling -> ruling.y1).orElse(0F);
+        double maxX = pages.stream().mapToDouble(ClassificationPage::getPageWidth).max().orElse(Double.MAX_VALUE);
+
+        return new Rectangle(new Point((float) maxX, (float) maxY), (float) 0, (float) minY, -1);
+    }
+*/
+

    /**
     * Adjusts and sets the body text frame to a page.
@@ -34,7 +94,7 @@ public class BodyTextFrameService {
     * @param bodyTextFrame          frame that contains the main text on portrait pages
     * @param landscapeBodyTextFrame frame that contains the main text on landscape pages
     */
-    public void setBodyTextFrameAdjustedToPage(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
+    private void setBodyTextFrameAdjustedToPage(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {

        Rectangle textFrame = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;

@@ -69,7 +129,10 @@ public class BodyTextFrameService {
     * @param landscape               Calculate for landscape or portrait
     * @return Rectangle of the text frame
     */
-    public Rectangle calculateBodyTextFrame(List<ClassificationPage> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape, LayoutParsingType layoutParsingType) {
+    private Rectangle calculateBodyTextFrame(List<ClassificationPage> pages,
+                                             FloatFrequencyCounter documentFontSizeCounter,
+                                             boolean landscape,
+                                             LayoutParsingType layoutParsingType) {

        float approximateHeaderLineCount;
        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
@@ -95,8 +158,8 @@ public class BodyTextFrameService {
                    }

                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10)
-                            || !layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount){
+                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10) || !layoutParsingType.equals(
+                            LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount) {
                        continue;
                    }

@@ -21,9 +21,9 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTrans
 import lombok.experimental.UtilityClass;

@UtilityClass
-public class TextPositionSequenceSorter {
+public class PageContentExtractor {

-    public List<PageContents> getSortedTextPositionsWithPages(String filename) throws IOException {
+    public List<PageContents> getSortedPageContents(String filename) throws IOException {

        List<PageContents> textPositionSequencesPerPage = new LinkedList<>();
        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
@@ -49,7 +49,8 @@ public class TextPositionSequenceSorter {

                    textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
                            RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
-                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox())));
+                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox()),
+                            stripper.getRulings()));
                }
            }
        }
@@ -1,151 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
-
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class PdfParsingService {
-
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final TaasBlockificationService taasBlockificationService;
-    private final DocuMineBlockificationService docuMineBlockificationService;
-    private final RedactManagerBlockificationService redactManagerBlockificationService;
-
-
-    public ClassificationDocument parseDocument(LayoutParsingType layoutParsingType,
-                                                PDDocument originDocument,
-                                                Map<Integer, List<TableCells>> pdfTableCells,
-                                                Map<Integer, List<ClassifiedImage>> pdfImages) {
-
-        ClassificationDocument document = new ClassificationDocument();
-        List<ClassificationPage> classificationPages = new ArrayList<>();
-
-        originDocument.setAllSecurityToBeRemoved(true);
-        long pageCount = originDocument.getNumberOfPages();
-
-        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-            parsePage(layoutParsingType, pdfImages, originDocument, pdfTableCells, document, classificationPages, pageNumber);
-        }
-
-        document.setPages(classificationPages);
-
-        return document;
-    }
-
-
-    @SneakyThrows
-    private void parsePage(LayoutParsingType layoutParsingType,
-                           Map<Integer, List<ClassifiedImage>> pdfImages,
-                           PDDocument pdDocument,
-                           Map<Integer, List<TableCells>> pdfTableCells,
-                           ClassificationDocument document,
-                           List<ClassificationPage> classificationPages,
-                           int pageNumber) {
-
-        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-        stripper.setPageNumber(pageNumber);
-        stripper.setStartPage(pageNumber);
-        stripper.setEndPage(pageNumber);
-        stripper.setPdpage(pdPage);
-        stripper.getText(pdDocument);
-
-        PDRectangle pdr = pdPage.getMediaBox();
-
-        int rotation = pdPage.getRotation();
-        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-        PDRectangle cropbox = pdPage.getCropBox();
-        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                stripper.getRulings(),
-                stripper.getMinCharWidth(),
-                stripper.getMaxCharHeight());
-
-        ClassificationPage classificationPage = switch (layoutParsingType) {
-            case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-        };
-
-        classificationPage.setRotation(rotation);
-        classificationPage.setLandscape(isLandscape);
-        classificationPage.setPageNumber(pageNumber);
-        classificationPage.setPageWidth(cropbox.getWidth());
-        classificationPage.setPageHeight(cropbox.getHeight());
-
-        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-            classificationPage.setImages(pdfImages.get(pageNumber));
-            imageServiceResponseAdapter.findOcr(classificationPage);
-        }
-
-        tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
-        buildPageStatistics(classificationPage);
-        increaseDocumentStatistics(classificationPage, document);
-
-        classificationPages.add(classificationPage);
-    }
-
-
-    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
-
-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-        }
-        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
-        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
-        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
-    }
-
-
-    private void buildPageStatistics(ClassificationPage classificationPage) {
-
-        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
-        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
-            if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getSequences() == null) {
-                    continue;
-                }
-                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
-                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
-                    classificationPage.getFontCounter().add(word.getFont());
-                    classificationPage.getFontSizeCounter().add(word.getFontSize());
-                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
-                }
-            }
-        }
-
-    }
-
-}
-
-
@@ -5,6 +5,9 @@ import static java.util.stream.Collectors.toSet;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

@@ -23,6 +26,8 @@ public class DocuMineBlockificationService {

    static final float THRESHOLD = 1f;

+    Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);
+

    /**
     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
@@ -43,7 +48,6 @@ public class DocuMineBlockificationService {
        float maxX = 0;
        float minY = 1000;
        float maxY = 0;
-
        TextPositionSequence prev = null;

        boolean wasSplitted = false;
@@ -60,7 +64,10 @@ public class DocuMineBlockificationService {
            boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle()
                    .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));

-            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap)) {
+            Matcher matcher = pattern.matcher(chunkWords.stream().collect(Collectors.joining(" ")).toString());
+            boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();
+
+            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap || startsOnSameX)) {

                Orientation prevOrientation = null;
                if (!chunkBlockList1.isEmpty()) {
@@ -231,3 +238,4 @@ public class DocuMineBlockificationService {
    }

 }
+
@@ -7,41 +7,33 @@ import java.util.regex.Pattern;

 import org.springframework.stereotype.Service;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

-
@Slf4j
@Service
@RequiredArgsConstructor
 public class DocuMineClassificationService {

-    private final BodyTextFrameService bodyTextFrameService;
-    private static final Pattern pattern = Pattern.compile("^(\\d{1,1}\\.?){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z\\[\\]\\-]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern2 = Pattern.compile(".*\\d{4}$", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern2 = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
    private static final Pattern pattern3 = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");


    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.DOCUMINE);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.DOCUMINE);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
-            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
            classifyPage(page, document, headlineFontSizes);
        }
    }
@@ -5,14 +5,11 @@ import java.util.regex.Pattern;

 import org.springframework.stereotype.Service;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -23,19 +20,14 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RedactManagerClassificationService {

-    private final BodyTextFrameService bodyTextFrameService;
-

    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.REDACT_MANAGER);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.REDACT_MANAGER);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
-            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
            classifyPage(page, document, headlineFontSizes);
        }
    }
@@ -5,8 +5,6 @@ import java.util.regex.Pattern;

 import org.springframework.stereotype.Service;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -28,14 +26,13 @@ public class TaasClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.TAAS);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.TAAS);
+
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
-            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
+
            classifyPage(page, document, headlineFontSizes);
        }
    }
@@ -32,7 +32,12 @@ public class TableNodeFactory {
        setPageNumberInCells(tablesToMerge);
        Set<Page> pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet());
        List<List<Cell>> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList();
-        Table table = Table.builder().documentTree(context.getDocumentTree()).numberOfCols(mergedRows.get(0).size()).numberOfRows(mergedRows.size()).build();
+
+        Table table = Table.builder()
+                .documentTree(context.getDocumentTree())
+                .numberOfCols(mergedRows.isEmpty() ? 0 : mergedRows.get(0).size())
+                .numberOfRows(mergedRows.size())
+                .build();

        pages.forEach(page -> addTableToPage(page, parentNode, table));

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
@@ -87,7 +88,8 @@ public class PropertiesMapper {

    public static String toString(Rectangle2D rectangle2D) {

-        return String.format("%f%s%f%s%f%s%f",
+        return String.format(Locale.US,
+                "%f%s%f%s%f%s%f",
                rectangle2D.getX(),
                DocumentStructure.RECTANGLE_DELIMITER,
                rectangle2D.getY(),
@@ -12,7 +12,6 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.apache.pdfbox.pdmodel.font.Standard14Fonts;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
@@ -72,7 +71,7 @@ public class PdfVisualisationUtility {

        contentStream.beginText();
        contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
-        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
+        contentStream.setFont(PDType1Font.HELVETICA, 10);
        contentStream.showText(string);
        contentStream.endText();
        contentStream.close();
@@ -152,7 +151,7 @@ public class PdfVisualisationUtility {


    @SneakyThrows
-    public static void drawLine2DList(PDDocument pdDocument, int pageNumber, List<Line2D> line2DS, Options options) {
+    public static void drawLine2DList(PDDocument pdDocument, int pageNumber, List<? extends Line2D> line2DS, Options options) {

        var pdPage = pdDocument.getPage(pageNumber - 1);
        var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true);
@@ -4,22 +4,25 @@ plugins {
    application
    id("com.knecon.fforesight.java-conventions")
    id("org.springframework.boot") version "3.1.2"
-    id("io.spring.dependency-management") version "1.1.0"
-    id("org.sonarqube") version "4.2.1.3168"
+    id("io.spring.dependency-management") version "1.1.2"
+    id("org.sonarqube") version "4.3.0.3225"
    id("io.freefair.lombok") version "8.1.0"
 //    id("org.graalvm.buildtools.native") version "0.9.23"
 }

 description = "layoutparser-service-server"

+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0-alpha2"
+
 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.iqser.red.commons:storage-commons:2.29.0")
    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
-    implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
+    implementation("com.amazonaws:aws-java-sdk-s3:1.12.526")
    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")

@@ -27,12 +30,12 @@ dependencies {
    testImplementation(project(":layoutparser-service-internal-api"))

    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
-    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
-    testImplementation("com.iqser.red.commons:jackson-commons:1.0.0")
-    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
-    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
-    testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
-    testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
+    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.124.0")
+    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
+    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
    testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
 }

@@ -67,6 +70,33 @@ sonarqube {
 }

 tasks.named<BootBuildImage>("bootBuildImage") {
+
+//    environment.put("BP_ACCEPT_EULA","Y")
+////    println("--------------------" + layout.projectDirectory.file( "src/main/resources/Aptfile").toString());
+//    var aptfilePath = layout.projectDirectory.file( "src/main/resources/Aptfile").toString() // make sure this path is valid because there will be no log or exception if not
+//    bindings.add("${aptfilePath}:/workspace/Aptfile:ro")
+//    buildpacks.set(listOf("ghcr.io/fagiani/buildpacks/fagiani_apt@sha256:6471c8c70f32b749e29f65ae562ac0339fecad26aa9217628c00a6c31f197dae", "urn:cnb:builder:paketo-buildpacks/java"))
+
+
+//    environment.put("BP_LOG_LEVEL","DEBUG")
+//    var aptfilePath = layout.projectDirectory.file( "src/main/resources/Aptfile").toString() // make sure this path is valid because there will be no log or exception if not
+//    bindings.add("${aptfilePath}:/workspace/Aptfile:ro")
+//    var shfilePath = layout.projectDirectory.file( "src/main/resources/buildpack-run.sh").toString() // make sure this path is valid because there will be no log or exception if not
+//    bindings.add("${shfilePath}:/workspace/buildpack-run.sh:ro")
+//    buildpacks.set(listOf("ghcr.io/fagiani/buildpacks/fagiani_run@sha256:5c75bf651072e1ad345aa148433b8cdd373e97f9b29523ceaeaf1c39f1691985", "ghcr.io/fagiani/buildpacks/fagiani_apt@sha256:6471c8c70f32b749e29f65ae562ac0339fecad26aa9217628c00a6c31f197dae","urn:cnb:builder:paketo-buildpacks/java"))
+
+    var fontsfilePath = layout.projectDirectory.file( "src/main/resources/fonts.tar.gz").toString() // make sure this path is valid because there will be no log or exception if not
+    bindings.add("${fontsfilePath}:/workspace/fonts.tar.gz:ro")
+
+    var shfilePath = layout.projectDirectory.file( "src/main/resources/buildpack-run.sh").toString() // make sure this path is valid because there will be no log or exception if not
+    bindings.add("${shfilePath}:/workspace/buildpack-run.sh:ro")
+    buildpacks.set(listOf("ghcr.io/fagiani/buildpacks/fagiani_run@sha256:5c75bf651072e1ad345aa148433b8cdd373e97f9b29523ceaeaf1c39f1691985", "urn:cnb:builder:paketo-buildpacks/java"))
+
+
+
+
+
+
    imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
    if (project.hasProperty("buildbootDockerHostNetwork")) {
        network.set("host")
@@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;

 import lombok.RequiredArgsConstructor;
@@ -34,6 +35,9 @@ public class MessageHandler {

        LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);

+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS) && layoutParsingRequest.researchDocumentStorageId() == null) {
+            throw new IllegalArgumentException("ResearchDocumentDataStorageId is null!");
+        }
        log.info("Layout parsing request received {}", layoutParsingRequest.identifier());
        if (message.getMessageProperties().isRedelivered()) {
            throw new AmqpRejectAndDontRequeueException(String.format("Error during last layout parsing of request with identifier: %s, do not retry.",
@@ -0,0 +1,14 @@
+# you can list packages
+ghostscript
+cabextract
+xfonts-utils
+fonts-liberation
+lmodern
+ttf-mscorefonts-installer
+
+
+# or include links to specific .deb files
+# http://ftp.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb
+
+# or add custom apt repos (only required if using packages outside of the standard Ubuntu APT repositories)
+# :repo:deb http://cz.archive.ubuntu.com/ubuntu artful main universe
@@ -0,0 +1 @@
+tar -xf fonts.tar.gz
@@ -22,10 +22,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
@@ -46,7 +47,7 @@ public class BdrJsonBuildTest extends AbstractTest {

        try (InputStream inputStream = new FileInputStream(filename)) {
            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
-                return layoutParsingPipeline.parseLayoutWithTimer(LayoutParsingType.TAAS, pdDocument, new ImageServiceResponse(), new TableServiceResponse());
+                return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS, pdDocument, new ImageServiceResponse(), new TableServiceResponse()));
            }
        }
    }
@@ -34,6 +34,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;

 import lombok.AllArgsConstructor;
@@ -94,10 +95,10 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

-        Document documentGraph = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                Loader.loadPDF(pdfFileResource.getInputStream()),
                new ImageServiceResponse(),
-                new TableServiceResponse());
+                new TableServiceResponse()));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -15,6 +15,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;

 import lombok.SneakyThrows;
@@ -29,7 +30,7 @@ public class BuildDocumentGraphTest extends AbstractTest {
    @Disabled
    public void buildMetolachlor() {

-        Document documentGraph = buildGraph("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
+        Document documentGraph = buildGraph("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
        assertEquals(221, documentGraph.getPages().size());
        assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
        assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
@@ -47,10 +48,10 @@ public class BuildDocumentGraphTest extends AbstractTest {
        ClassPathResource fileResource = new ClassPathResource(filename);

        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream)) {
-            return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
                    pdDocument,
                    layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
-                    new TableServiceResponse());
+                    new TableServiceResponse()));
        }
    }

@@ -18,6 +18,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;

 import lombok.SneakyThrows;
@@ -54,10 +55,10 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentGraphTest {
    @SneakyThrows
    private void writeJsons(Path filename) {

-        Document documentGraph = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                Loader.loadPDF(new FileInputStream(filename.toFile())),
                new ImageServiceResponse(),
-                new TableServiceResponse());
+                new TableServiceResponse()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.server.graph;
 import java.awt.Color;
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;

 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -23,7 +24,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
    @Disabled
    public void visualizeMetolachlor() {

-        String filename = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
+        String filename = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
        visualizePdf(filename);
    }

@@ -33,7 +34,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
    @Disabled
    public void visualizeRotatedTestDocument() {

-        String filename = "files/211";
+        String filename = "files/new/RotateTestFile.pdf";
        visualizePdf(filename);
    }

@@ -43,7 +44,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
    @Disabled
    public void visualizeCraftedDocument() {

-        String filename = "files/crafted document";
+        String filename = "files/crafted document.pdf";
        visualizePdf(filename);
    }

@@ -60,8 +61,8 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {

    private static void visualizeSemanticNodes(String filename, Document documentGraph, TextBlock textBlock) throws IOException {

-        File tmpFile = File.createTempFile(filename, "SEMANTIC_NODES_BBOX.pdf");
-        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+        File tmpFile = new File("/tmp/" + Path.of(filename).getFileName().toString() + "_SEMANTIC_NODES_BBOX.pdf");
+        ClassPathResource fileResource = new ClassPathResource(filename);

        try (var fileStream = fileResource.getInputStream();//
             PDDocument pdDocument = Loader.loadPDF(fileStream)//
@@ -21,16 +21,16 @@ import org.springframework.core.io.ClassPathResource;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
@@ -40,7 +40,7 @@ import lombok.SneakyThrows;
 public class PdfSegmentationServiceTest extends AbstractTest {

    @Autowired
-    private PdfParsingService pdfParsingService;
+    private LayoutParsingPipeline layoutParsingPipeline;

    @Autowired
    private ObjectMapper objectMapper;
@@ -57,12 +57,13 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @Autowired
    private SectionsBuilderService sectionsBuilderService;

+
    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {

-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(LayoutParsingType.REDACT_MANAGER,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(new TableServiceResponse()),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(new ImageServiceResponse()));
+                new ImageServiceResponse(),
+                new TableServiceResponse());

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -14,7 +14,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.DividingCol
 import com.knecon.fforesight.service.layoutparser.processor.services.GapDetectionService;
 import com.knecon.fforesight.service.layoutparser.processor.services.GapsAcrossLinesService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;
@@ -30,7 +30,7 @@ class GapAcrossLinesDetectionServiceTest {
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_COLUMNS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
-        List<PageInformation> pageInformations = TextPositionSequenceSorter.getSortedTextPositionsWithPages(filename).stream().map(PageInformationService::build).toList();
+        List<PageInformation> pageInformations = PageContentExtractor.getSortedPageContents(filename).stream().map(PageInformationService::build).toList();
        List<List<Rectangle2D>> columnsPerPage = new LinkedList<>();
        System.out.printf("Finished TextPosition Extraction in %d ms%n", System.currentTimeMillis() - start);
        System.out.println("start column detection");
@@ -56,7 +56,7 @@ class GapAcrossLinesDetectionServiceTest {
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_COLUMNS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
-        List<PageContents> sortedTextPositionSequencesPerPage = TextPositionSequenceSorter.getSortedTextPositionsWithPages(filename);
+        List<PageContents> sortedTextPositionSequencesPerPage = PageContentExtractor.getSortedPageContents(filename);
        List<List<Rectangle2D>> columnsPerPage = new LinkedList<>();
        System.out.printf("Finished TextPosition Extraction in %d ms%n", System.currentTimeMillis() - start);
        System.out.println("start column detection");
@@ -13,7 +13,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.PageInformatio
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.InvisibleTableDetectionService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

@@ -28,7 +28,7 @@ class InvisibleTableDetectionServiceTest {

        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TABLE.pdf").toString();
-        List<PageInformation> pageContents = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName).stream().map(PageInformationService::build).collect(Collectors.toList());
+        List<PageInformation> pageContents = PageContentExtractor.getSortedPageContents(fileName).stream().map(PageInformationService::build).collect(Collectors.toList());

        int pageNumber = 1;
        Rectangle2D tableBBox = pageContents.get(0)
@@ -7,7 +7,7 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;

 import lombok.SneakyThrows;

@@ -20,7 +20,7 @@ class MainBodyTextFrameExtractionServiceTest {

        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
        String tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_MAIN_BODY.pdf").toString();
-        List<PageContents> sortedTextPositionSequence = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName);
+        List<PageContents> sortedTextPositionSequence = PageContentExtractor.getSortedPageContents(fileName);


    }
@@ -9,12 +9,12 @@ import org.junit.jupiter.api.Test;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;

-class TextPositionSequenceSorterTest {
+class PageContentExtractorTest {

    @Test
    @Disabled
@@ -24,7 +24,7 @@ class TextPositionSequenceSorterTest {
        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TEXT_POSITION_SEQUENCES.pdf").toString();

-        List<PageContents> textPositionPerPage = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName);
+        List<PageContents> textPositionPerPage = PageContentExtractor.getSortedPageContents(fileName);

        PdfDraw.drawRectanglesPerPageNumberedByLine(fileName,
                textPositionPerPage.stream()
@@ -8,7 +8,7 @@ import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageInformation;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;
@@ -24,7 +24,7 @@ class PageInformationServiceTest {
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_GAPS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
-        List<PageInformation> pageInformations = TextPositionSequenceSorter.getSortedTextPositionsWithPages(filename).stream().map(PageInformationService::build).toList();
+        List<PageInformation> pageInformations = PageContentExtractor.getSortedPageContents(filename).stream().map(PageInformationService::build).toList();
        System.out.printf("Finished TextPosition Extraction in %d ms%n", System.currentTimeMillis() - start);
        System.out.println("start gap detection");
        start = System.currentTimeMillis();
@@ -47,7 +47,7 @@ class PageInformationServiceTest {
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_GAPS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
-        List<PageInformation> pageInformations = TextPositionSequenceSorter.getSortedTextPositionsWithPages(filename).stream().map(PageInformationService::build).toList();
+        List<PageInformation> pageInformations = PageContentExtractor.getSortedPageContents(filename).stream().map(PageInformationService::build).toList();
        System.out.printf("Finished TextPosition Extraction in %d ms%n", System.currentTimeMillis() - start);
        System.out.println("start gap detection");
        start = System.currentTimeMillis();
@@ -0,0 +1,38 @@
+package com.knecon.fforesight.service.layoutparser.server.services;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
+
+import lombok.SneakyThrows;
+
+public class RulingCleaningServiceTest {
+
+    @Test
+    @Disabled
+    @SneakyThrows
+    public void textRulingExtraction() {
+
+        String fileName = "files/BASF/2013-1110704.pdf";
+        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
+        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents("files/BASF/2013-1110704.pdf");
+        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
+
+        RulingCleaningService rulingCleaningService = new RulingCleaningService();
+        List<CleanRulings> cleanRulingsPerPage = new LinkedList<>();
+        for (PageContents pageContent : pageContents) {
+            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings(), 8, 20));
+        }
+    }
+
+}
@@ -20,6 +20,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;

 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
 import com.knecon.fforesight.service.layoutparser.server.Application;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -52,18 +53,23 @@ public abstract class AbstractTest {
    protected final static String POSITION_FILE_ID = "positions";
    protected final static String PAGES_FILE_ID = "pages";
    protected final static String TENANT_ID = "tenant";
+    protected final static String SECTION_GRID_ID = "section";
+    protected final static String SIMPLIFIED_ID = "simplified";


    protected LayoutParsingRequest buildStandardLayoutParsingRequest() {

        return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                .build();
    }

@@ -94,13 +100,16 @@ public abstract class AbstractTest {

        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileInputStream);
        return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.empty())
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                .originFileStorageId(ORIGIN_FILE_ID)
-                .tablesFileStorageId(Optional.empty())
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                .build();
    }

@@ -124,13 +133,16 @@ public abstract class AbstractTest {
        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);

        return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                .build();
    }

@@ -13,7 +13,6 @@ import java.util.Map;
 import java.util.stream.Collectors;

 import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -36,14 +35,14 @@ public class FileSystemBackedStorageService implements StorageService {

    @SneakyThrows
    @Override
-    public InputStreamResource getObject(String tenantId, String objectId) {
+    public void downloadTo(String tenantId, String objectId, File destinationFile) {

        var res = dataMap.get(objectId);
        if (res == null) {
            throw new StorageObjectDoesNotExist(new RuntimeException());
        }
-        return new InputStreamResource(new FileInputStream(res));

+        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
    }


@@ -24,6 +24,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

@@ -206,6 +207,21 @@ public class PdfDraw {
    }


+    @SneakyThrows
+    public static void drawLinesPerPage(String filename, List<List<Ruling>> linesPerPage, String tmpFileName) {
+
+        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
+             PDDocument pdDocument = Loader.loadPDF(inputStream);//
+             var out = new FileOutputStream(tmpFileName)//
+        ) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                PdfVisualisationUtility.drawLine2DList(pdDocument, pageNumber, linesPerPage.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
+            }
+            pdDocument.save(out);
+        }
+    }
+
+
    @Builder
    @AllArgsConstructor
    @Getter
@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}
Author	SHA1	Message	Date
deiflaender	b1eb946b87	hotfix: test to add fonts	2023-08-10 10:55:25 +02:00
Renovate Bot	24104f8cc1	Merge branch 'renovate/main-com.iqser.red.commons-storage-commons-2.x' into 'main' Update dependency com.iqser.red.commons:storage-commons to v2.29.0 (main) See merge request fforesight/layout-parser!17	2023-08-10 09:14:03 +02:00
Renovate Bot	3632dd4667	Update dependency com.iqser.red.commons:storage-commons to v2.29.0	2023-08-10 04:14:44 +00:00
Renovate Bot	063aa8bfe1	Merge branch 'renovate/main-com.iqser.red.commons-jackson-commons-1.x' into 'main' Update dependency com.iqser.red.commons:jackson-commons to v1.3.0 (main) See merge request fforesight/layout-parser!16	2023-08-10 06:14:25 +02:00
Renovate Bot	d2716a60e9	Update dependency com.iqser.red.commons:jackson-commons to v1.3.0	2023-08-10 01:13:56 +00:00
Renovate Bot	8f08a8c62b	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.526 (main) See merge request fforesight/layout-parser!22	2023-08-10 03:13:38 +02:00
Renovate Bot	091cb73622	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.526	2023-08-09 22:14:12 +00:00
Renovate Bot	d3b0bc430f	Merge branch 'renovate/main-plugins-(non-major)' into 'main' Update Plugins (non-major) (main) See merge request fforesight/layout-parser!15	2023-08-10 00:13:58 +02:00
Renovate Bot	e3c12bc1bb	Update Plugins (non-major)	2023-08-09 19:14:15 +00:00
Renovate Bot	f6f7a0a952	Merge branch 'renovate/main-jacksonversion' into 'main' Update jacksonVersion to v2.15.2 (main) See merge request fforesight/layout-parser!13	2023-08-09 21:13:57 +02:00
Renovate Bot	96df6e3145	Update jacksonVersion to v2.15.2	2023-08-09 16:12:54 +00:00
Renovate Bot	574e5ad425	Merge branch 'renovate/main-aws-java-sdk-monorepo' into 'main' Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.525 (main) See merge request fforesight/layout-parser!12	2023-08-09 18:12:37 +02:00
Renovate Bot	0611e56baa	Update dependency com.amazonaws:aws-java-sdk-s3 to v1.12.525	2023-08-09 13:13:33 +00:00
Kilian Schüttler	442c1dafea	Merge branch 'update-pdfbox' into 'main' update PDFBox Version See merge request fforesight/layout-parser!19	2023-08-09 12:48:14 +02:00
Kilian Schüttler	33bc532eac	Merge branch 'renovate/main-com.iqser.red.service-persistence-service-shared-api-v1-2.x' into 'main' Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.124.0 (main) See merge request fforesight/layout-parser!18	2023-08-09 12:42:38 +02:00
Kilian Schuettler	4bd6e7e343	update PDFBox Version	2023-08-09 12:41:28 +02:00
Renovate Bot	159ac6348c	Update dependency com.iqser.red.service:persistence-service-shared-api-v1 to v2.124.0	2023-08-09 10:14:27 +00:00
Kilian Schuettler	17259ed805	add renovate, fix checkstyle	2023-08-09 10:11:02 +02:00
Dominique Eifländer	67bf5cbaa8	Merge branch 'RED-6725' into 'main' RED-6725: Install fonts See merge request fforesight/layout-parser!11	2023-08-09 10:07:05 +02:00
deiflaender	f8a3cbc147	RED-6725: Install fonts	2023-08-09 10:00:07 +02:00
Andrei Isvoran	a3d4fbe3a3	Merge branch 'RED-6864' into 'main' RED-6864 - Switch to DELETE_ON_CLOSE See merge request fforesight/layout-parser!10	2023-08-09 08:37:07 +02:00
Andrei Isvoran	5c1dca5933	RED-6864 - Switch to DELETE_ON_CLOSE	2023-08-09 09:30:37 +03:00
Timo Bejan	f56ab8fa49	Merge branch 'RED-6864' into 'main' RED-6864 - Switch to new storage-commons download See merge request fforesight/layout-parser!9	2023-08-08 17:16:40 +02:00
Andrei Isvoran	cfca5376a0	RED-6864 - Switch to new storage-commons download	2023-08-08 17:16:40 +02:00
Kevin Tumma	0633fa04fb	Update file .gitlab-ci.yml	2023-08-08 13:00:40 +02:00
Dominique Eifländer	659a9abaa5	Merge branch 'DM-165' into 'main' DM-165: Fixed numberFormatException on german local machines See merge request fforesight/layout-parser!8	2023-08-07 12:29:37 +02:00
deiflaender	5877aea3f7	DM-165: Fixed numberFormatException on german local machines	2023-08-07 12:12:00 +02:00
deiflaender	f2b92de827	DM-165: Fixed indexOutOfBounds error in TableNodeFactory	2023-08-05 10:20:05 +02:00
Kilian Schuettler	4a5464d6aa	Refactoring to make downstream refactoring easier	2023-08-04 15:16:36 +02:00
Dominique Eifländer	d9a3bbbd30	Merge branch 'RED-5253' into 'main' RED-5253: Ported last documine changes See merge request fforesight/layout-parser!7	2023-08-04 09:59:57 +02:00
deiflaender	150aea55c0	RED-5253: Ported last documine changes	2023-08-04 09:55:35 +02:00
Kilian Schuettler	676f0c9d09	cleanup dependency versions	2023-08-01 10:27:14 +02:00
Kilian Schuettler	ded00df11e	fix build	2023-08-01 09:57:58 +02:00
Kilian Schuettler	286556cbb6	mark non nullable fields in request	2023-08-01 00:50:33 +02:00