Merge branch 'RED-5253' into 'main'

RED-5253: Ported last documine changes

See merge request fforesight/layout-parser!7

config/pmd/pmd.xml

@@ -12,7 +12,6 @@
         <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
         <exclude name="NullAssignment"/>
-        <exclude name="BeanMembersShouldSerialize"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
         <exclude name="AvoidFieldNameMatchingMethodName"/>

config/pmd/test_pmd.xml

@@ -13,7 +13,6 @@
         <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
         <exclude name="NullAssignment"/>
-        <exclude name="BeanMembersShouldSerialize"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
         <exclude name="AvoidFieldNameMatchingMethodName"/>

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.awt.geom.Rectangle2D;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -21,6 +23,40 @@ public class DocumentStructure {
     EntryData root;
 
 
+    public static class TableProperties {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    public static class ImageProperties {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+    }
+
+    public static class TableCellProperties {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
     public EntryData get(List<Integer> tocId) {
 
         if (tocId.isEmpty()) {
@@ -66,6 +102,7 @@ public class DocumentStructure {
         Map<String, String> properties;
         List<EntryData> children;
 
+
         @Override
         public String toString() {
 

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedSectionText.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import java.util.ArrayList;
 import java.util.List;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -4,20 +4,21 @@ import java.util.Map;
 import java.util.Optional;
 
 import lombok.Builder;
-
+import lombok.NonNull;
 
 @Builder
 public record LayoutParsingRequest(
-        LayoutParsingType layoutParsingType,
+        @NonNull LayoutParsingType layoutParsingType,
         Map<String, String> identifier,
-        String originFileStorageId,
+        @NonNull String originFileStorageId,
         Optional<String> tablesFileStorageId,
         Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
+        @NonNull String structureFileStorageId,
         String researchDocumentStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId,
-        String sectionGridStorageId) {
+        @NonNull String textBlockFileStorageId,
+        @NonNull String positionBlockFileStorageId,
+        @NonNull String pageFileStorageId,
+        @NonNull String simplifiedTextStorageId,
+        @NonNull String sectionGridStorageId) {
 
 }

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -5,6 +5,9 @@ plugins {
 
 description = "layoutparser-service-processor"
 
+val jacksonVersion = "2.15.0-rc2"
+val pdfBoxVersion = "3.0.0-alpha2"
+
 dependencies {
     implementation(project(":layoutparser-service-internal-api"))
 
@@ -15,9 +18,9 @@ dependencies {
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
     implementation("com.iqser.red.commons:storage-commons:2.1.0")
 
-    implementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
-    implementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
-    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
-    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
+    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -10,10 +10,12 @@ import org.springframework.stereotype.Service;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
@@ -43,6 +45,7 @@ public class LayoutParsingPipeline {
     private final TaasClassificationService taasClassificationService;
     private final RedactManagerClassificationService redactManagerClassificationService;
     private final DocuMineClassificationService docuMineClassificationService;
+    private final SimplifiedSectionTextService simplifiedSectionTextService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -52,12 +55,12 @@ public class LayoutParsingPipeline {
         try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
             ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
             if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
             }
 
             TableServiceResponse tableServiceResponse = new TableServiceResponse();
             if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
             }
 
             Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
@@ -65,7 +68,7 @@ public class LayoutParsingPipeline {
 
             layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
             layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-
+            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
             if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
                 var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
                 layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -18,6 +18,7 @@ import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -110,4 +111,10 @@ public class LayoutParsingStorageService {
         }
     }
 
+
+    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Classification.java

@@ -3,9 +3,15 @@ package com.knecon.fforesight.service.layoutparser.processor.python_api.model.im
 import java.util.HashMap;
 import java.util.Map;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Classification {
 
     private Map<String, Float> probabilities = new HashMap<>();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/FilterGeometry.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class FilterGeometry {
 
     private ImageSize imageSize;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Filters.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Filters {
 
     private FilterGeometry geometry;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Geometry.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Geometry {
 
     private float width;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageFormat.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageFormat {
 
     private float quotient;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageMetadata.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageMetadata {
 
     private Classification classification;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageServiceResponse.java

@@ -6,9 +6,15 @@ import java.util.List;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageServiceResponse {
 
     private String dossierId;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageSize.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageSize {
 
     private float quotient;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Position.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Position {
 
     private float x1;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Probability.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Probability {
 
     private boolean unconfident;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/PageInfo.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class PageInfo {
 
     private int number;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/PdfTableCell.java

@@ -3,12 +3,12 @@ package com.knecon.fforesight.service.layoutparser.processor.python_api.model.ta
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.RequiredArgsConstructor;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@RequiredArgsConstructor
 public class PdfTableCell {
 
     private float x0;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableCells.java

@@ -1,10 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
 
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class TableCells {
 
     private float x0;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableData.java

@@ -3,9 +3,15 @@ package com.knecon.fforesight.service.layoutparser.processor.python_api.model.ta
 import java.util.ArrayList;
 import java.util.List;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class TableData {
 
     private PageInfo pageInfo;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableServiceResponse.java

@@ -3,9 +3,15 @@ package com.knecon.fforesight.service.layoutparser.processor.python_api.model.ta
 import java.util.ArrayList;
 import java.util.List;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class TableServiceResponse {
 
     private String dossierId;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PdfParsingService.java

@@ -77,6 +77,9 @@ public class PdfParsingService {
         stripper.setStartPage(pageNumber);
         stripper.setEndPage(pageNumber);
         stripper.setPdpage(pdPage);
+        if(layoutParsingType.equals(LayoutParsingType.DOCUMINE)){
+            stripper.setSortByPosition(true);
+        }
         stripper.getText(pdDocument);
 
         PDRectangle pdr = pdPage.getMediaBox();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -0,0 +1,26 @@
+package com.knecon.fforesight.service.layoutparser.processor.services;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+
+@Service
+public class SimplifiedSectionTextService {
+
+    public SimplifiedText toSimplifiedText(Document document) {
+
+        List<SimplifiedSectionText> simplifiedSectionTexts = document.getMainSections().stream().map(this::toSimplifiedSectionText).toList();
+        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build();
+    }
+
+
+    private SimplifiedSectionText toSimplifiedSectionText(Section section) {
+
+        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
+    }
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocuMineBlockificationService.java

@@ -5,6 +5,9 @@ import static java.util.stream.Collectors.toSet;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
@@ -23,6 +26,8 @@ public class DocuMineBlockificationService {
 
     static final float THRESHOLD = 1f;
 
+    Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);
+
 
     /**
      * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
@@ -39,11 +44,7 @@ public class DocuMineBlockificationService {
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractPageBlock> chunkBlockList1 = new ArrayList<>();
 
-        float minX = 1000;
-        float maxX = 0;
-        float minY = 1000;
-        float maxY = 0;
-
+        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
         TextPositionSequence prev = null;
 
         boolean wasSplitted = false;
@@ -60,7 +61,10 @@ public class DocuMineBlockificationService {
             boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle()
                     .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
 
-            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap)) {
+            Matcher matcher = pattern.matcher(chunkWords.stream().collect(Collectors.joining(" ")).toString());
+            boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();
+
+            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap || startsOnSameX)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {
@@ -231,3 +235,4 @@ public class DocuMineBlockificationService {
     }
 
 }
+

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/DocuMineClassificationService.java

@@ -20,15 +20,14 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class DocuMineClassificationService {
 
     private final BodyTextFrameService bodyTextFrameService;
-    private static final Pattern pattern = Pattern.compile("^(\\d{1,1}\\.?){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z\\[\\]\\-]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern2 = Pattern.compile(".*\\d{4}$", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern2 = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
     private static final Pattern pattern3 = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java

@@ -1,36 +1,24 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
 
 import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 
 public class PropertiesMapper {
 
-    public static final String TRANSPARENT = "transparent";
-    public static final String IMAGE_TYPE = "imageType";
-    public static final String POSITION = "position";
-    public static final String ROW = "row";
-    public static final String COL = "col";
-    public static final String HEADER = "header";
-    public static final String B_BOX = "bBox";
-    public static final String NUMBER_OF_ROWS = "numberOfRows";
-    public static final String NUMBER_OF_COLS = "numberOfCols";
-
-
     public static Map<String, String> buildImageProperties(Image image) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(IMAGE_TYPE, image.getImageType().toString());
-        properties.put(TRANSPARENT, String.valueOf(image.isTransparent()));
-        properties.put(POSITION, RectangleTransformations.toString(image.getPosition()));
+        properties.put(DocumentStructure.ImageProperties.IMAGE_TYPE, image.getImageType().toString());
+        properties.put(DocumentStructure.ImageProperties.TRANSPARENT, String.valueOf(image.isTransparent()));
+        properties.put(DocumentStructure.ImageProperties.POSITION, toString(image.getPosition()));
+        properties.put(DocumentStructure.ImageProperties.ID, image.getId());
         return properties;
     }
 
@@ -38,15 +26,15 @@ public class PropertiesMapper {
     public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(ROW, String.valueOf(tableCell.getRow()));
-        properties.put(COL, String.valueOf(tableCell.getCol()));
-        properties.put(HEADER, String.valueOf(tableCell.isHeader()));
+        properties.put(DocumentStructure.TableCellProperties.ROW, String.valueOf(tableCell.getRow()));
+        properties.put(DocumentStructure.TableCellProperties.COL, String.valueOf(tableCell.getCol()));
+        properties.put(DocumentStructure.TableCellProperties.HEADER, String.valueOf(tableCell.isHeader()));
 
         if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
             throw new IllegalArgumentException("TableCell can only occur on a single page!");
         }
-        String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
-        properties.put(B_BOX, bBoxString);
+        String bBoxString = toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
+        properties.put(DocumentStructure.TableCellProperties.B_BOX, bBoxString);
 
         return properties;
     }
@@ -55,33 +43,33 @@ public class PropertiesMapper {
     public static Map<String, String> buildTableProperties(Table table) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
-        properties.put(NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
+        properties.put(DocumentStructure.TableProperties.NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
+        properties.put(DocumentStructure.TableProperties.NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
         return properties;
     }
 
 
     public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
 
-        builder.imageType(parseImageType(properties.get(IMAGE_TYPE)));
-        builder.transparent(Boolean.parseBoolean(properties.get(TRANSPARENT)));
-        builder.position(parseRectangle2D(properties.get(POSITION)));
+        builder.imageType(parseImageType(properties.get(DocumentStructure.ImageProperties.IMAGE_TYPE)));
+        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructure.ImageProperties.TRANSPARENT)));
+        builder.position(DocumentStructure.parseRectangle2D(properties.get(DocumentStructure.ImageProperties.POSITION)));
     }
 
 
     public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
 
-        builder.row(Integer.parseInt(properties.get(ROW)));
-        builder.col(Integer.parseInt(properties.get(COL)));
-        builder.header(Boolean.parseBoolean(properties.get(HEADER)));
-        builder.bBox(parseRectangle2D(properties.get(B_BOX)));
+        builder.row(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.ROW)));
+        builder.col(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.COL)));
+        builder.header(Boolean.parseBoolean(properties.get(DocumentStructure.TableCellProperties.HEADER)));
+        builder.bBox(DocumentStructure.parseRectangle2D(properties.get(DocumentStructure.TableCellProperties.B_BOX)));
     }
 
 
     public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
 
-        builder.numberOfRows(Integer.parseInt(properties.get(NUMBER_OF_ROWS)));
-        builder.numberOfCols(Integer.parseInt(properties.get(NUMBER_OF_COLS)));
+        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_ROWS)));
+        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_COLS)));
     }
 
 
@@ -99,14 +87,14 @@ public class PropertiesMapper {
 
     public static String toString(Rectangle2D rectangle2D) {
 
-        return String.format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+        return String.format("%f%s%f%s%f%s%f",
+                rectangle2D.getX(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getY(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getWidth(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getHeight());
     }
 
 }

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -12,6 +12,9 @@ plugins {
 
 description = "layoutparser-service-server"
 
+val jacksonVersion = "2.15.0-rc2"
+val pdfBoxVersion = "3.0.0-alpha2"
+
 dependencies {
     implementation(project(":layoutparser-service-processor"))
     implementation(project(":layoutparser-service-internal-api"))
@@ -21,6 +24,7 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
     implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
     implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
+    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
 
     // for integration testing only
     testImplementation(project(":layoutparser-service-internal-api"))
@@ -28,10 +32,10 @@ dependencies {
     testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
     testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
     testImplementation("com.iqser.red.commons:jackson-commons:1.0.0")
-    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
-    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
-    testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
-    testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
+    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
     testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
 }
 

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 
 import lombok.RequiredArgsConstructor;
@@ -34,6 +35,9 @@ public class MessageHandler {
 
         LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);
 
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS) && layoutParsingRequest.researchDocumentStorageId() == null) {
+            throw new IllegalArgumentException("ResearchDocumentDataStorageId is null!");
+        }
         log.info("Layout parsing request received {}", layoutParsingRequest.identifier());
         if (message.getMessageProperties().isRedelivered()) {
             throw new AmqpRejectAndDontRequeueException(String.format("Error during last layout parsing of request with identifier: %s, do not retry.",

layoutparser-service/layoutparser-service-server/src/main/resources/application.yml

@@ -2,6 +2,7 @@ info:
   description: Layout Parser Service Processor
 
 tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
+
 fforesight.tenants.remote: true
 
 server:

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/ApplicationTests.java

@@ -2,9 +2,9 @@ package com.knecon.fforesight.service.layoutparser.server;
 
 import org.junit.jupiter.api.Test;
 
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
-class ApplicationTests extends BaseTest {
+class ApplicationTests extends AbstractTest {
 
     @Test
     void contextLoads() {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -27,12 +27,12 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
 
 import lombok.SneakyThrows;
 
-public class BdrJsonBuildTest extends BaseTest {
+public class BdrJsonBuildTest extends AbstractTest {
 
     @Autowired
     private ObjectMapper objectMapper;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/BuildDocumentGraphTest.java

@@ -15,11 +15,11 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
 import lombok.SneakyThrows;
 
-public class BuildDocumentGraphTest extends BaseTest {
+public class BuildDocumentGraphTest extends AbstractTest {
 
     @Autowired
     protected LayoutParsingPipeline layoutParsingPipeline;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -33,11 +33,11 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePag
 import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
 import lombok.SneakyThrows;
 
-public class PdfSegmentationServiceTest extends BaseTest {
+public class PdfSegmentationServiceTest extends AbstractTest {
 
     @Autowired
     private PdfParsingService pdfParsingService;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -20,6 +20,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
 import com.knecon.fforesight.service.layoutparser.server.Application;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -29,8 +30,8 @@ import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(BaseTest.TestConfiguration.class)
-public class BaseTest {
+@Import(AbstractTest.TestConfiguration.class)
+public abstract class AbstractTest {
 
     @Autowired
     protected LayoutParsingStorageService layoutParsingStorageService;
@@ -52,18 +53,23 @@ public class BaseTest {
     protected final static String POSITION_FILE_ID = "positions";
     protected final static String PAGES_FILE_ID = "pages";
     protected final static String TENANT_ID = "tenant";
+    protected final static String SECTION_GRID_ID = "section";
+    protected final static String SIMPLIFIED_ID = "simplified";
 
 
     protected LayoutParsingRequest buildStandardLayoutParsingRequest() {
 
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
                 .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }
 
@@ -94,13 +100,16 @@ public class BaseTest {
 
         storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileInputStream);
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.empty())
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
-                .tablesFileStorageId(Optional.empty())
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }
 
@@ -124,13 +133,16 @@ public class BaseTest {
         storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);
 
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
                 .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }