Merge branch 'RED-5253' into 'main'

RED-5253: Ported last documine changes

See merge request fforesight/layout-parser!7

config/pmd/pmd.xml

@@ -12,7 +12,6 @@
         <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
         <exclude name="NullAssignment"/>
-        <exclude name="BeanMembersShouldSerialize"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
         <exclude name="AvoidFieldNameMatchingMethodName"/>

config/pmd/test_pmd.xml

@@ -13,7 +13,6 @@
         <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
         <exclude name="NullAssignment"/>
-        <exclude name="BeanMembersShouldSerialize"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
         <exclude name="AvoidFieldNameMatchingMethodName"/>

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -35,6 +35,7 @@ public class DocumentStructure {
         public static final String TRANSPARENT = "transparent";
         public static final String IMAGE_TYPE = "imageType";
         public static final String POSITION = "position";
+        public static final String ID = "id";
 
     }
 

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -4,21 +4,21 @@ import java.util.Map;
 import java.util.Optional;
 
 import lombok.Builder;
-
+import lombok.NonNull;
 
 @Builder
 public record LayoutParsingRequest(
-        LayoutParsingType layoutParsingType,
+        @NonNull LayoutParsingType layoutParsingType,
         Map<String, String> identifier,
-        String originFileStorageId,
+        @NonNull String originFileStorageId,
         Optional<String> tablesFileStorageId,
         Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
+        @NonNull String structureFileStorageId,
         String researchDocumentStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId,
-        String simplifiedTextStorageId,
-        String sectionGridStorageId) {
+        @NonNull String textBlockFileStorageId,
+        @NonNull String positionBlockFileStorageId,
+        @NonNull String pageFileStorageId,
+        @NonNull String simplifiedTextStorageId,
+        @NonNull String sectionGridStorageId) {
 
 }

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -5,6 +5,9 @@ plugins {
 
 description = "layoutparser-service-processor"
 
+val jacksonVersion = "2.15.0-rc2"
+val pdfBoxVersion = "3.0.0-alpha2"
+
 dependencies {
     implementation(project(":layoutparser-service-internal-api"))
 
@@ -15,9 +18,9 @@ dependencies {
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
     implementation("com.iqser.red.commons:storage-commons:2.1.0")
 
-    implementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
-    implementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
-    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
-    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
+    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PdfParsingService.java

@@ -77,6 +77,9 @@ public class PdfParsingService {
         stripper.setStartPage(pageNumber);
         stripper.setEndPage(pageNumber);
         stripper.setPdpage(pdPage);
+        if(layoutParsingType.equals(LayoutParsingType.DOCUMINE)){
+            stripper.setSortByPosition(true);
+        }
         stripper.getText(pdDocument);
 
         PDRectangle pdr = pdPage.getMediaBox();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocuMineBlockificationService.java

@@ -5,6 +5,9 @@ import static java.util.stream.Collectors.toSet;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
@@ -23,6 +26,8 @@ public class DocuMineBlockificationService {
 
     static final float THRESHOLD = 1f;
 
+    Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);
+
 
     /**
      * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
@@ -39,11 +44,7 @@ public class DocuMineBlockificationService {
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractPageBlock> chunkBlockList1 = new ArrayList<>();
 
-        float minX = 1000;
-        float maxX = 0;
-        float minY = 1000;
-        float maxY = 0;
-
+        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
         TextPositionSequence prev = null;
 
         boolean wasSplitted = false;
@@ -60,7 +61,10 @@ public class DocuMineBlockificationService {
             boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle()
                     .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
 
-            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap)) {
+            Matcher matcher = pattern.matcher(chunkWords.stream().collect(Collectors.joining(" ")).toString());
+            boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();
+
+            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap || startsOnSameX)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {
@@ -231,3 +235,4 @@ public class DocuMineBlockificationService {
     }
 
 }
+

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/DocuMineClassificationService.java

@@ -20,15 +20,14 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class DocuMineClassificationService {
 
     private final BodyTextFrameService bodyTextFrameService;
-    private static final Pattern pattern = Pattern.compile("^(\\d{1,1}\\.?){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z\\[\\]\\-]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern2 = Pattern.compile(".*\\d{4}$", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern2 = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
     private static final Pattern pattern3 = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java

@@ -18,6 +18,7 @@ public class PropertiesMapper {
         properties.put(DocumentStructure.ImageProperties.IMAGE_TYPE, image.getImageType().toString());
         properties.put(DocumentStructure.ImageProperties.TRANSPARENT, String.valueOf(image.isTransparent()));
         properties.put(DocumentStructure.ImageProperties.POSITION, toString(image.getPosition()));
+        properties.put(DocumentStructure.ImageProperties.ID, image.getId());
         return properties;
     }
 

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -12,6 +12,9 @@ plugins {
 
 description = "layoutparser-service-server"
 
+val jacksonVersion = "2.15.0-rc2"
+val pdfBoxVersion = "3.0.0-alpha2"
+
 dependencies {
     implementation(project(":layoutparser-service-processor"))
     implementation(project(":layoutparser-service-internal-api"))
@@ -29,10 +32,10 @@ dependencies {
     testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
     testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
     testImplementation("com.iqser.red.commons:jackson-commons:1.0.0")
-    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
-    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
-    testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
-    testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
+    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
     testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
 }
 

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 
 import lombok.RequiredArgsConstructor;
@@ -34,6 +35,9 @@ public class MessageHandler {
 
         LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);
 
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS) && layoutParsingRequest.researchDocumentStorageId() == null) {
+            throw new IllegalArgumentException("ResearchDocumentDataStorageId is null!");
+        }
         log.info("Layout parsing request received {}", layoutParsingRequest.identifier());
         if (message.getMessageProperties().isRedelivered()) {
             throw new AmqpRejectAndDontRequeueException(String.format("Error during last layout parsing of request with identifier: %s, do not retry.",

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/ApplicationTests.java

@@ -2,9 +2,9 @@ package com.knecon.fforesight.service.layoutparser.server;
 
 import org.junit.jupiter.api.Test;
 
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
-class ApplicationTests extends BaseTest {
+class ApplicationTests extends AbstractTest {
 
     @Test
     void contextLoads() {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -27,12 +27,12 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
 
 import lombok.SneakyThrows;
 
-public class BdrJsonBuildTest extends BaseTest {
+public class BdrJsonBuildTest extends AbstractTest {
 
     @Autowired
     private ObjectMapper objectMapper;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/BuildDocumentGraphTest.java

@@ -15,11 +15,11 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
 import lombok.SneakyThrows;
 
-public class BuildDocumentGraphTest extends BaseTest {
+public class BuildDocumentGraphTest extends AbstractTest {
 
     @Autowired
     protected LayoutParsingPipeline layoutParsingPipeline;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -33,11 +33,11 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePag
 import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 
 import lombok.SneakyThrows;
 
-public class PdfSegmentationServiceTest extends BaseTest {
+public class PdfSegmentationServiceTest extends AbstractTest {
 
     @Autowired
     private PdfParsingService pdfParsingService;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -20,6 +20,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
 import com.knecon.fforesight.service.layoutparser.server.Application;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -29,8 +30,8 @@ import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(BaseTest.TestConfiguration.class)
-public class BaseTest {
+@Import(AbstractTest.TestConfiguration.class)
+public abstract class AbstractTest {
 
     @Autowired
     protected LayoutParsingStorageService layoutParsingStorageService;
@@ -52,18 +53,23 @@ public class BaseTest {
     protected final static String POSITION_FILE_ID = "positions";
     protected final static String PAGES_FILE_ID = "pages";
     protected final static String TENANT_ID = "tenant";
+    protected final static String SECTION_GRID_ID = "section";
+    protected final static String SIMPLIFIED_ID = "simplified";
 
 
     protected LayoutParsingRequest buildStandardLayoutParsingRequest() {
 
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
                 .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }
 
@@ -94,13 +100,16 @@ public class BaseTest {
 
         storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileInputStream);
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.empty())
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
-                .tablesFileStorageId(Optional.empty())
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }
 
@@ -124,13 +133,16 @@ public class BaseTest {
         storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);
 
         return LayoutParsingRequest.builder()
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
                 .originFileStorageId(ORIGIN_FILE_ID)
                 .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .pageFileStorageId(PAGES_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
                 .structureFileStorageId(STRUCTURE_FILE_ID)
                 .textBlockFileStorageId(TEXT_FILE_ID)
+                .positionBlockFileStorageId(POSITION_FILE_ID)
+                .pageFileStorageId(PAGES_FILE_ID)
+                .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .build();
     }