RED-8481: Use visual layout parsing to detect signatures

implemented visuallayoutparsingresult

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -18,11 +18,15 @@ public record LayoutParsingRequest(
 
         @Schema(description = "Path to the original PDF file.")//
         @NonNull String originFileStorageId,//
+
+
         @Schema(description = "Optional Path to the table extraction file.")//
         Optional<String> tablesFileStorageId,//
         @Schema(description = "Optional Path to the image classification file.")//
         Optional<String> imagesFileStorageId,//
 
+        @Schema(description = "Optional Path to the the visual layout parsing service file") Optional<String> visualLayoutParsingFileId,
+
         @Schema(description = "Path where the Document Structure File will be stored.")//
         @NonNull String structureFileStorageId,//
         @Schema(description = "Path where the Research Data File will be stored.")//

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -33,9 +33,12 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageB
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
@@ -86,6 +89,7 @@ public class LayoutParsingPipeline {
     RedactManagerBlockificationService redactManagerBlockificationService;
     LayoutGridService layoutGridService;
     ObservationRegistry observationRegistry;
+    VisualLayoutParsingAdapter visualLayoutParsingAdapter;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -96,6 +100,11 @@ public class LayoutParsingPipeline {
         File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
         File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);
 
+        VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse();
+        if (layoutParsingRequest.visualLayoutParsingFileId().isPresent()) {
+            visualLayoutParsingResponse = layoutParsingStorageService.getExtractedTablesFile(layoutParsingRequest.visualLayoutParsingFileId().get());
+        }
+
         ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
         if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
             imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
@@ -110,6 +119,7 @@ public class LayoutParsingPipeline {
                 originFile,
                 imageServiceResponse,
                 tableServiceResponse,
+                visualLayoutParsingResponse,
                 layoutParsingRequest.identifier().toString());
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
@@ -198,12 +208,14 @@ public class LayoutParsingPipeline {
                                               File originFile,
                                               ImageServiceResponse imageServiceResponse,
                                               TableServiceResponse tableServiceResponse,
+                                              VisualLayoutParsingResponse visualLayoutParsingResponse,
                                               String identifier) {
 
         PDDocument originDocument = openDocument(originFile);
         addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
         Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
         Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
+        Map<Integer, List<ClassifiedImage>> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse);
 
         ClassificationDocument classificationDocument = new ClassificationDocument();
         List<ClassificationPage> classificationPages = new ArrayList<>();
@@ -264,6 +276,10 @@ public class LayoutParsingPipeline {
                 imageServiceResponseAdapter.findOcr(classificationPage);
             }
 
+            if(signatures != null && signatures.containsKey(pageNumber)) {
+                classificationPage.setImages(signatures.get(pageNumber));
+            }
+
             tableExtractionService.extractTables(cleanRulings, classificationPage);
 
             buildPageStatistics(classificationPage);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -22,6 +22,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.Researc
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
@@ -79,6 +80,15 @@ public class LayoutParsingStorageService {
         }
     }
 
+    public VisualLayoutParsingResponse getExtractedTablesFile(String storageId) throws IOException {
+
+        try (InputStream inputStream = getObject(storageId)) {
+            VisualLayoutParsingResponse visualLayoutParsingResponse = objectMapper.readValue(inputStream, VisualLayoutParsingResponse.class);
+            inputStream.close();
+            return visualLayoutParsingResponse;
+        }
+    }
+
 
     @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
     public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java

@@ -0,0 +1,83 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingBox;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class VisualLayoutParsingAdapter {
+
+    public Map<Integer, List<VisualLayoutParsingResult>> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
+
+        Map<Integer, List<VisualLayoutParsingResult>> tableCells = new HashMap<>();
+        visualLayoutParsingResponse.getData()
+                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertTableCells(tableData.getBoxes())));
+
+        return tableCells;
+
+    }
+
+    public Map<Integer, List<ClassifiedImage>> buildExtractedSignaturesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
+
+        Map<Integer, List<ClassifiedImage>> tableCells = new HashMap<>();
+        visualLayoutParsingResponse.getData()
+                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertSignatures(tableData.getPage_idx(),tableData.getBoxes())));
+
+        return tableCells;
+
+    }
+
+
+    public List<VisualLayoutParsingResult> convertTableCells(List<VisualLayoutParsingBox> tableObjects) {
+
+        List<VisualLayoutParsingResult> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            VisualLayoutParsingResult result = new VisualLayoutParsingResult();
+            result.setX0(t.getBox().getX1());
+            result.setX1(t.getBox().getX2());
+            result.setY0(t.getBox().getY1());
+            result.setY1(t.getBox().getY2());
+            result.setWidth(result.getX1() - result.getX0());
+            result.setHeight(result.getY1() - result.getY0());
+            result.setLabel(t.getLabel());
+            parsedTableCells.add(result);
+        });
+
+        return parsedTableCells;
+
+    }
+
+    public List<ClassifiedImage> convertSignatures(int pageNumber, List<VisualLayoutParsingBox> tableObjects) {
+
+        List<ClassifiedImage> signatures = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            if(t.getLabel().equals("signature")) {
+                ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(),t.getBox().getY1(),t.getBox().getX2() - t.getBox().getX1(),t.getBox().getY2() - t.getBox().getY1()),
+                        ImageType.SIGNATURE,false,pageNumber);
+
+                signatures.add(signature);
+            }
+        });
+
+        return signatures;
+
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBox.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingBox {
+
+    private com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingBoxValue box;
+    private String label;
+    private float probability;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBoxValue.java

@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingBoxValue {
+
+    private float x1;
+    private float y1;
+    private float x2;
+    private float y2;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingData.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingData {
+
+    private int page_idx;
+
+    private List<VisualLayoutParsingBox> boxes;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResponse.java

@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingResponse {
+
+    private String dossierId;
+    private String fileId;
+    private String targetFileExtension;
+    private String responseFileExtension;
+    private String X_TENANT_ID;
+    private List<VisualLayoutParsingData> data;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResult.java

@@ -0,0 +1,22 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingResult {
+
+    private float x0;
+    private float y0;
+    private float x1;
+    private float y1;
+    private float width;
+    private float height;
+    private String label;
+
+}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -50,6 +50,7 @@ public class BdrJsonBuildTest extends AbstractTest {
                 file,
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
+                null,
                 file.toString()));
     }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -98,6 +98,7 @@ public class HeadlinesGoldStandardIntegrationTest {
                 pdfFileResource.getFile(),
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
+                null,
                 filePath));
 
         var foundHeadlines = documentGraph.streamAllSubNodes()

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphJsonWritingTest.java

@@ -58,6 +58,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
                 filename.toFile(),
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
+                null,
                 filename.toFile().toString()));
 
         DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -51,7 +51,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
         var tableResponse = mapper.readValue(new ClassPathResource(tableFileName).getInputStream(), TableServiceResponse.class);
         var documentFile = new ClassPathResource(fileName).getFile();
 
-        var classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE, documentFile, new ImageServiceResponse(), tableResponse, Path.of(fileName).getFileName().toFile().toString());
+        var classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE, documentFile, new ImageServiceResponse(), tableResponse, null,Path.of(fileName).getFileName().toFile().toString());
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
         Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -67,6 +67,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                 originDocument,
                 new ImageServiceResponse(),
                 tableServiceResponse,
+                null,
                 "document");
 
         redactManagerClassificationService.classifyDocument(classificationDocument);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -80,11 +80,13 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                 filename.toFile(),
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
+                null,
                 filename.toFile().toString()));
         Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                 filename.toFile(),
                 new ImageServiceResponse(),
                 new TableServiceResponse(),
+                null,
                 filename.toFile().toString()));
         DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
         DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -25,7 +25,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
 
         File fileResource = new ClassPathResource(filename).getFile();
         prepareStorage(filename);
-        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), filename);
+        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), null,filename);
     }