fix build

include openfeign for tenant-commons

See merge request fforesight/layout-parser!6

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -1,24 +1,16 @@
 plugins {
-    java
+    `java-library`
+    `maven-publish`
     pmd
     checkstyle
     jacoco
 }
 
 group = "com.knecon.fforesight"
-version = "0.1-SNAPSHOT"
 
 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17
 
-tasks.jacocoTestReport {
-    reports {
-        xml.required.set(false)
-        csv.required.set(false)
-        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
-    }
-}
-
 tasks.pmdMain {
     pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
 }
@@ -29,6 +21,9 @@ tasks.pmdTest {
 
 tasks.named<Test>("test") {
     useJUnitPlatform()
+    reports {
+        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
+    }
 }
 
 tasks.test {
@@ -40,14 +35,38 @@ tasks.jacocoTestReport {
     reports {
         xml.required.set(true)
         csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
     }
 }
 
+allprojects {
+    publishing {
+        publications {
+            create<MavenPublication>(name) {
+                from(components["java"])
+            }
+        }
+        repositories {
+            maven {
+                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+                credentials {
+                    username = providers.gradleProperty("mavenUser").getOrNull();
+                    password = providers.gradleProperty("mavenPassword").getOrNull();
+                }
+            }
+        }
+    }
+}
+
+java {
+    withJavadocJar()
+}
+
 repositories {
     mavenLocal()
     mavenCentral()
     maven {
-        url = uri("https://nexus.knecon.com/repository/gindev/");
+        url = uri("https://nexus.knecon.com/repository/gindev/")
         credentials {
             username = providers.gradleProperty("mavenUser").getOrNull();
             password = providers.gradleProperty("mavenPassword").getOrNull();

config/pmd/pmd.xml

@@ -12,7 +12,6 @@
         <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
         <exclude name="NullAssignment"/>
-        <exclude name="BeanMembersShouldSerialize"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
         <exclude name="AvoidFieldNameMatchingMethodName"/>

config/pmd/test_pmd.xml

@@ -13,7 +13,6 @@
         <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
         <exclude name="NullAssignment"/>
-        <exclude name="BeanMembersShouldSerialize"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
         <exclude name="AvoidFieldNameMatchingMethodName"/>

gradle.properties.kts

@@ -1 +1 @@
-version = 0.1.0
+version = 0.1-SNAPSHOT

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.awt.geom.Rectangle2D;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -21,6 +23,40 @@ public class DocumentStructure {
     EntryData root;
 
 
+    public static class TableProperties {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    public static class ImageProperties {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+    }
+
+    public static class TableCellProperties {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
     public EntryData get(List<Integer> tocId) {
 
         if (tocId.isEmpty()) {
@@ -66,6 +102,7 @@ public class DocumentStructure {
         Map<String, String> properties;
         List<EntryData> children;
 
+
         @Override
         public String toString() {
 

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedSectionText.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import java.util.ArrayList;
 import java.util.List;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -4,20 +4,21 @@ import java.util.Map;
 import java.util.Optional;
 
 import lombok.Builder;
-
+import lombok.NonNull;
 
 @Builder
 public record LayoutParsingRequest(
-        LayoutParsingType layoutParsingType,
+        @NonNull LayoutParsingType layoutParsingType,
         Map<String, String> identifier,
-        String originFileStorageId,
+        @NonNull String originFileStorageId,
         Optional<String> tablesFileStorageId,
         Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
+        @NonNull String structureFileStorageId,
         String researchDocumentStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId,
-        String sectionGridStorageId) {
+        @NonNull String textBlockFileStorageId,
+        @NonNull String positionBlockFileStorageId,
+        @NonNull String pageFileStorageId,
+        @NonNull String simplifiedTextStorageId,
+        @NonNull String sectionGridStorageId) {
 
 }

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -3,10 +3,15 @@ plugins {
     id("io.freefair.lombok") version "8.1.0"
 }
 
+description = "layoutparser-service-processor"
+
 dependencies {
     implementation(project(":layoutparser-service-internal-api"))
 
-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
+    }
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
     implementation("com.iqser.red.commons:storage-commons:2.1.0")
 
@@ -15,7 +20,4 @@ dependencies {
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
     implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
 }
-
-description = "layoutparser-service-processor"

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -10,10 +10,12 @@ import org.springframework.stereotype.Service;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
@@ -43,6 +45,7 @@ public class LayoutParsingPipeline {
     private final TaasClassificationService taasClassificationService;
     private final RedactManagerClassificationService redactManagerClassificationService;
     private final DocuMineClassificationService docuMineClassificationService;
+    private final SimplifiedSectionTextService simplifiedSectionTextService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -52,12 +55,12 @@ public class LayoutParsingPipeline {
         try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
             ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
             if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
             }
 
             TableServiceResponse tableServiceResponse = new TableServiceResponse();
             if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
             }
 
             Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
@@ -65,7 +68,7 @@ public class LayoutParsingPipeline {
 
             layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
             layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-
+            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
             if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
                 var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
                 layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -15,13 +15,10 @@ import org.springframework.stereotype.Service;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -90,26 +87,6 @@ public class LayoutParsingStorageService {
     }
 
 
-    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
-
-        DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class);
-        DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.textBlockFileStorageId(),
-                DocumentTextData[].class);
-        DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.positionBlockFileStorageId(),
-                DocumentPositionData[].class);
-        DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class);
-
-        return DocumentData.builder()
-                .documentStructure(tableOfContentsData)
-                .documentPositions(atomicPositionBlockData)
-                .documentTextData(documentTextDataBlockData)
-                .documentPages(documentPageData)
-                .build();
-    }
-
-
     private File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {
 
         File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();
@@ -134,4 +111,10 @@ public class LayoutParsingStorageService {
         }
     }
 
+
+    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Classification.java

@@ -3,9 +3,15 @@ package com.knecon.fforesight.service.layoutparser.processor.python_api.model.im
 import java.util.HashMap;
 import java.util.Map;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Classification {
 
     private Map<String, Float> probabilities = new HashMap<>();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/FilterGeometry.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class FilterGeometry {
 
     private ImageSize imageSize;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Filters.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Filters {
 
     private FilterGeometry geometry;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Geometry.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Geometry {
 
     private float width;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageFormat.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageFormat {
 
     private float quotient;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageMetadata.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageMetadata {
 
     private Classification classification;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageServiceResponse.java

@@ -6,9 +6,15 @@ import java.util.List;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageServiceResponse {
 
     private String dossierId;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageSize.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageSize {
 
     private float quotient;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Position.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Position {
 
     private float x1;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Probability.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class Probability {
 
     private boolean unconfident;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/PageInfo.java

@@ -1,8 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class PageInfo {
 
     private int number;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/PdfTableCell.java

@@ -3,12 +3,12 @@ package com.knecon.fforesight.service.layoutparser.processor.python_api.model.ta
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.RequiredArgsConstructor;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@RequiredArgsConstructor
 public class PdfTableCell {
 
     private float x0;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableCells.java

@@ -1,10 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
 
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class TableCells {
 
     private float x0;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableData.java

@@ -3,9 +3,15 @@ package com.knecon.fforesight.service.layoutparser.processor.python_api.model.ta
 import java.util.ArrayList;
 import java.util.List;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class TableData {
 
     private PageInfo pageInfo;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableServiceResponse.java

@@ -3,9 +3,15 @@ package com.knecon.fforesight.service.layoutparser.processor.python_api.model.ta
 import java.util.ArrayList;
 import java.util.List;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class TableServiceResponse {
 
     private String dossierId;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessagingConfiguration.java

@@ -1,40 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.queue;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
-
-import org.springframework.amqp.core.Queue;
-import org.springframework.amqp.core.QueueBuilder;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-import lombok.RequiredArgsConstructor;
-
-@Configuration
-@RequiredArgsConstructor
-public class MessagingConfiguration {
-
-    @Bean
-    public Queue layoutparsingRequestQueue() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingResponseQueue() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingDLQ() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -0,0 +1,26 @@
+package com.knecon.fforesight.service.layoutparser.processor.services;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+
+@Service
+public class SimplifiedSectionTextService {
+
+    public SimplifiedText toSimplifiedText(Document document) {
+
+        List<SimplifiedSectionText> simplifiedSectionTexts = document.getMainSections().stream().map(this::toSimplifiedSectionText).toList();
+        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build();
+    }
+
+
+    private SimplifiedSectionText toSimplifiedSectionText(Section section) {
+
+        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
+    }
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java

@@ -1,36 +1,24 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
 
 import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 
 public class PropertiesMapper {
 
-    public static final String TRANSPARENT = "transparent";
-    public static final String IMAGE_TYPE = "imageType";
-    public static final String POSITION = "position";
-    public static final String ROW = "row";
-    public static final String COL = "col";
-    public static final String HEADER = "header";
-    public static final String B_BOX = "bBox";
-    public static final String NUMBER_OF_ROWS = "numberOfRows";
-    public static final String NUMBER_OF_COLS = "numberOfCols";
-
-
     public static Map<String, String> buildImageProperties(Image image) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(IMAGE_TYPE, image.getImageType().toString());
-        properties.put(TRANSPARENT, String.valueOf(image.isTransparent()));
-        properties.put(POSITION, RectangleTransformations.toString(image.getPosition()));
+        properties.put(DocumentStructure.ImageProperties.IMAGE_TYPE, image.getImageType().toString());
+        properties.put(DocumentStructure.ImageProperties.TRANSPARENT, String.valueOf(image.isTransparent()));
+        properties.put(DocumentStructure.ImageProperties.POSITION, toString(image.getPosition()));
+        properties.put(DocumentStructure.ImageProperties.ID, image.getId());
         return properties;
     }
 
@@ -38,15 +26,15 @@ public class PropertiesMapper {
     public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(ROW, String.valueOf(tableCell.getRow()));
-        properties.put(COL, String.valueOf(tableCell.getCol()));
-        properties.put(HEADER, String.valueOf(tableCell.isHeader()));
+        properties.put(DocumentStructure.TableCellProperties.ROW, String.valueOf(tableCell.getRow()));
+        properties.put(DocumentStructure.TableCellProperties.COL, String.valueOf(tableCell.getCol()));
+        properties.put(DocumentStructure.TableCellProperties.HEADER, String.valueOf(tableCell.isHeader()));
 
         if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
             throw new IllegalArgumentException("TableCell can only occur on a single page!");
         }
-        String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
-        properties.put(B_BOX, bBoxString);
+        String bBoxString = toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
+        properties.put(DocumentStructure.TableCellProperties.B_BOX, bBoxString);
 
         return properties;
     }
@@ -55,33 +43,33 @@ public class PropertiesMapper {
     public static Map<String, String> buildTableProperties(Table table) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put(NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
-        properties.put(NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
+        properties.put(DocumentStructure.TableProperties.NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
+        properties.put(DocumentStructure.TableProperties.NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
         return properties;
     }
 
 
     public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
 
-        builder.imageType(parseImageType(properties.get(IMAGE_TYPE)));
-        builder.transparent(Boolean.parseBoolean(properties.get(TRANSPARENT)));
-        builder.position(parseRectangle2D(properties.get(POSITION)));
+        builder.imageType(parseImageType(properties.get(DocumentStructure.ImageProperties.IMAGE_TYPE)));
+        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructure.ImageProperties.TRANSPARENT)));
+        builder.position(DocumentStructure.parseRectangle2D(properties.get(DocumentStructure.ImageProperties.POSITION)));
     }
 
 
     public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
 
-        builder.row(Integer.parseInt(properties.get(ROW)));
-        builder.col(Integer.parseInt(properties.get(COL)));
-        builder.header(Boolean.parseBoolean(properties.get(HEADER)));
-        builder.bBox(parseRectangle2D(properties.get(B_BOX)));
+        builder.row(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.ROW)));
+        builder.col(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.COL)));
+        builder.header(Boolean.parseBoolean(properties.get(DocumentStructure.TableCellProperties.HEADER)));
+        builder.bBox(DocumentStructure.parseRectangle2D(properties.get(DocumentStructure.TableCellProperties.B_BOX)));
     }
 
 
     public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
 
-        builder.numberOfRows(Integer.parseInt(properties.get(NUMBER_OF_ROWS)));
-        builder.numberOfCols(Integer.parseInt(properties.get(NUMBER_OF_COLS)));
+        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_ROWS)));
+        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_COLS)));
     }
 
 
@@ -99,14 +87,14 @@ public class PropertiesMapper {
 
     public static String toString(Rectangle2D rectangle2D) {
 
-        return String.format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+        return String.format("%f%s%f%s%f%s%f",
+                rectangle2D.getX(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getY(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getWidth(),
+                DocumentStructure.RECTANGLE_DELIMITER,
+                rectangle2D.getHeight());
     }
 
 }

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -1,23 +1,27 @@
 import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 
 plugins {
+    application
     id("com.knecon.fforesight.java-conventions")
     id("org.springframework.boot") version "3.1.2"
     id("io.spring.dependency-management") version "1.1.0"
     id("org.sonarqube") version "4.2.1.3168"
     id("io.freefair.lombok") version "8.1.0"
-    //id("org.graalvm.buildtools.native") version "0.9.23"
+//    id("org.graalvm.buildtools.native") version "0.9.23"
 }
 
+description = "layoutparser-service-server"
+
 dependencies {
     implementation(project(":layoutparser-service-processor"))
+    implementation(project(":layoutparser-service-internal-api"))
 
     implementation("com.iqser.red.commons:storage-commons:2.1.0")
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-
     implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
     implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
-
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
+    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
 
     // for integration testing only
     testImplementation(project(":layoutparser-service-internal-api"))
@@ -30,23 +34,17 @@ dependencies {
     testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
     testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
     testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
-    testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
-}
-
-description = "layoutparser-service-server"
-
-java {
-    withJavadocJar()
 }
 
 /*
-TODO: implement native build using GRAALVM, currently the problem is, logback needs runtime configuration using reflections or something
-kinda weird behavior, since the buildArgs below seem to have no effect at all
-https://stackoverflow.com/questions/72770461/graalvm-native-image-can-not-compile-logback-dependencies
+//TODO: implement native build using GRAALVM, currently the problem is, logback needs runtime configuration using reflections or something
+//kinda weird behavior, since the buildArgs below seem to have no effect at all
+//https://stackoverflow.com/questions/72770461/graalvm-native-image-can-not-compile-logback-dependencies
 
+We traced the issue to storage-commons and there to azure blob storage, this is today an open bug with azure spring
+https://github.com/Azure/azure-sdk-for-java/issues/36001
 
-// AOT seems to be the name of the generated classes for native images
-// They are added as a SourceSet, and therefore checkstyle and pmd try to run on it
+//AOT is springs ahead of time compilation, it is a code analysis which generates new code, therefore checkstyle and pmd can be disabled on them.
 tasks.named<Checkstyle>("checkstyleAot") {
     enabled = false
 }
@@ -59,28 +57,15 @@ tasks.named<Pmd>("pmdAot") {
 tasks.named<Pmd>("pmdAotTest") {
     enabled = false
 }
+*/
 
-graalvmNative {
-    binaries {
-        all {
-            resources.autodetect()
-            buildArgs.add("--verbose")
-            buildArgs.add("--add-opens=java.base/java.nio=ALL-UNNAMED")
-            buildArgs.add("--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED")
-            buildArgs.add("--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED")
-            buildArgs.add("--trace-class-initialization=ch.qos.logback.classic.Logger")
-            buildArgs.add("--trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase\$Worker")
-            buildArgs.add("--initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback")
-            buildArgs.add("--initialize-at-run-time=io.netty")
-        }
-
+sonarqube {
+    properties {
+        providers.gradleProperty("sonarToken").getOrNull()?.let { property("sonar.login", it) }
+        property("sonar.host.url", "https://sonarqube.knecon.com")
     }
 }
 
-nativeBuild {
-    buildArgs("-H:ReflectionConfigurationFiles=../../../src/main/resources/reflection-config.json")
-}
-*/
 tasks.named<BootBuildImage>("bootBuildImage") {
     imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
     if (project.hasProperty("buildbootDockerHostNetwork")) {

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/Application.java

@@ -10,7 +10,7 @@ import org.springframework.context.annotation.Import;
 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
-import com.knecon.fforesight.service.layoutparser.processor.queue.MessagingConfiguration;
+import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
 @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -1,7 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.queue;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
+package com.knecon.fforesight.service.layoutparser.server.queue;
 
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
@@ -12,7 +9,9 @@ import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 
 import lombok.RequiredArgsConstructor;
@@ -30,12 +29,15 @@ public class MessageHandler {
 
 
     @RabbitHandler
-    @RabbitListener(queues = LAYOUT_PARSING_REQUEST_QUEUE)
+    @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)
     @SneakyThrows
     public void receiveLayoutParsingRequest(Message message) {
 
         LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);
 
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS) && layoutParsingRequest.researchDocumentStorageId() == null) {
+            throw new IllegalArgumentException("ResearchDocumentDataStorageId is null!");
+        }
         log.info("Layout parsing request received {}", layoutParsingRequest.identifier());
         if (message.getMessageProperties().isRedelivered()) {
             throw new AmqpRejectAndDontRequeueException(String.format("Error during last layout parsing of request with identifier: %s, do not retry.",
@@ -49,7 +51,7 @@ public class MessageHandler {
 
     public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
 
-        rabbitTemplate.convertAndSend(LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
     }
 
 }

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java

@@ -0,0 +1,38 @@
+package com.knecon.fforesight.service.layoutparser.server.queue;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+
+import lombok.RequiredArgsConstructor;
+
+@Configuration
+@RequiredArgsConstructor
+public class MessagingConfiguration {
+
+    @Bean
+    public Queue layoutparsingRequestQueue() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    @Bean
+    public Queue layoutparsingResponseQueue() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    @Bean
+    public Queue layoutparsingDLQ() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/main/resources/application.yml

@@ -2,6 +2,7 @@ info:
   description: Layout Parser Service Processor
 
 tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
+
 fforesight.tenants.remote: true
 
 server:

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/ApplicationTests.java

@@ -2,9 +2,9 @@ package com.knecon.fforesight.service.layoutparser.server;
 
 import org.junit.jupiter.api.Test;
 
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
-class ApplicationTests extends BaseTest {
+class ApplicationTests extends AbstractTest {
 
     @Test
     void contextLoads() {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -27,12 +27,12 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
 
 import lombok.SneakyThrows;
 
-public class BdrJsonBuildTest extends BaseTest {
+public class BdrJsonBuildTest extends AbstractTest {
 
     @Autowired
     private ObjectMapper objectMapper;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/BuildDocumentGraphTest.java

@@ -15,11 +15,11 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
 import lombok.SneakyThrows;
 
-public class BuildDocumentGraphTest extends BaseTest {
+public class BuildDocumentGraphTest extends AbstractTest {
 
     @Autowired
     protected LayoutParsingPipeline layoutParsingPipeline;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -33,11 +33,11 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePag
 import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
 import lombok.SneakyThrows;
 
-public class PdfSegmentationServiceTest extends BaseTest {
+public class PdfSegmentationServiceTest extends AbstractTest {
 
     @Autowired
     private PdfParsingService pdfParsingService;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -20,6 +20,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
 import com.knecon.fforesight.service.layoutparser.server.Application;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -29,8 +30,8 @@ import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(BaseTest.TestConfiguration.class)
-public class BaseTest {
+@Import(AbstractTest.TestConfiguration.class)
+public abstract class AbstractTest {
 
     @Autowired
     protected LayoutParsingStorageService layoutParsingStorageService;
@@ -52,18 +53,23 @@ public class BaseTest {
     protected final static String POSITION_FILE_ID = "positions";
     protected final static String PAGES_FILE_ID = "pages";
     protected final static String TENANT_ID = "tenant";
+    protected final static String SECTION_GRID_ID = "section";
+    protected final static String SIMPLIFIED_ID = "simplified";
 
 
     protected LayoutParsingRequest buildStandardLayoutParsingRequest() {
 
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
                 .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }
 
@@ -94,13 +100,16 @@ public class BaseTest {
 
         storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileInputStream);
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.empty())
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
-                .tablesFileStorageId(Optional.empty())
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }
 
@@ -124,13 +133,16 @@ public class BaseTest {
         storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);
 
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
                 .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }