RED-7375 table extractor prototype

created new branch because old one broke

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -20,6 +20,9 @@ public record LayoutParsingRequest(
         @NonNull String originFileStorageId,//
         @Schema(description = "Optional Path to the table extraction file.")//
         Optional<String> tablesFileStorageId,//
+
+        @Schema(description= "Optional Path to the the table parsing service file")
+        Optional<String> tableExtractorFileId,//
         @Schema(description = "Optional Path to the image classification file.")//
         Optional<String> imagesFileStorageId,//
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -33,8 +33,11 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageB
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.TableExtractorResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
@@ -71,6 +74,7 @@ import lombok.extern.slf4j.Slf4j;
 public class LayoutParsingPipeline {
 
     ImageServiceResponseAdapter imageServiceResponseAdapter;
+    TableExtractorResponseAdapter tableExtractorResponseAdapter;
     CvTableParsingAdapter cvTableParsingAdapter;
     LayoutParsingStorageService layoutParsingStorageService;
     SectionsBuilderService sectionsBuilderService;
@@ -101,6 +105,11 @@ public class LayoutParsingPipeline {
             imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
         }
 
+        TableExtractorResponse tableExtractorResponse = new TableExtractorResponse();
+        if (layoutParsingRequest.tableExtractorFileId().isPresent()) {
+            tableExtractorResponse = layoutParsingStorageService.getExtractedTableFile(layoutParsingRequest.tableExtractorFileId().get());
+        }
+
         TableServiceResponse tableServiceResponse = new TableServiceResponse();
         if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
             tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
@@ -119,8 +128,10 @@ public class LayoutParsingPipeline {
         layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
         layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
 
+        Map<Integer, List<TableExtractorCells>> extractedTableCells = tableExtractorResponseAdapter.buildExtractedTablesPerPage(tableExtractorResponse);
+
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
-        viewerDocumentService.createViewerDocument(originFile, documentGraph, viewerDocumentFile, false);
+        viewerDocumentService.createViewerDocument(originFile, documentGraph, viewerDocumentFile, extractedTableCells,false);
         layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
         if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
             log.info("Building research document data for {}", layoutParsingRequest.identifier());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -20,6 +20,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Si
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
@@ -62,6 +63,14 @@ public class LayoutParsingStorageService {
         }
     }
 
+    public TableExtractorResponse getExtractedTableFile(String storageId) throws IOException {
+
+        try (InputStream inputStream = getObject(storageId)) {
+            TableExtractorResponse tableExtractorResponse = objectMapper.readValue(inputStream, TableExtractorResponse.class);
+            inputStream.close();
+            return tableExtractorResponse;
+        }
+    }
 
     public TableServiceResponse getTablesFile(String storageId) throws IOException {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -28,6 +28,7 @@ public class TextPositionSequence implements CharSequence {
 
     public static final int HEIGHT_PADDING = 2;
     private int page;
+    @Builder.Default
     private List<RedTextPosition> textPositions = new ArrayList<>();
 
     private TextDirection dir;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/TableExtractorResponseAdapter.java

@@ -0,0 +1,75 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTable;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTableData;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableData;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class TableExtractorResponseAdapter {
+
+    public Map<Integer, List<TableExtractorCells>> buildExtractedTablesPerPage(TableExtractorResponse tableExtractorResponse) {
+        Map<Integer, List<TableExtractorCells>> tableCells = new HashMap<>();
+        tableExtractorResponse.getData()
+                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_number(), tableCell -> new ArrayList<>())
+                        .addAll(convertTableCells(tableData.getTables())));
+
+        return tableCells;
+
+    }
+
+    public List<TableExtractorCells> convertTableCells(List<ExtractedTable> tableObjects) {
+
+        List<TableExtractorCells> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            TableExtractorCells tableCells = new TableExtractorCells();
+            tableCells.setX0(t.getTable().getBbox().get(0));
+            tableCells.setX1(t.getTable().getBbox().get(2));
+            tableCells.setY0(t.getTable().getBbox().get(1));
+            tableCells.setY1(t.getTable().getBbox().get(3));
+            tableCells.setWidth(tableCells.getX1()- tableCells.getX0());
+            tableCells.setHeight(tableCells.getY1()- tableCells.getY0());
+            tableCells.setLabel(t.getTable().getLabel());
+            log.info("Parsed table cell {} with label {}",tableCells, tableCells.getLabel());
+            parsedTableCells.add(tableCells);
+            t.getObjects().forEach(o -> {
+                TableExtractorCells objectCell = new TableExtractorCells();
+                objectCell.setX0(o.getBbox().get(0));
+                objectCell.setX1(o.getBbox().get(2));
+                objectCell.setY0(o.getBbox().get(1));
+                objectCell.setY1(o.getBbox().get(3));
+                objectCell.setWidth(objectCell.getX1()- objectCell.getX0());
+                objectCell.setHeight(objectCell.getY1()- objectCell.getY0());
+                objectCell.setLabel(o.getLabel());
+                log.info("Parsed object cell {} with label {}",objectCell, objectCell.getLabel());
+                parsedTableCells.add(objectCell);
+            });
+        });
+
+        log.info("result from parsing: {}",parsedTableCells);
+        return parsedTableCells;
+
+    }
+
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Classification.java

@@ -13,7 +13,7 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 @AllArgsConstructor
 public class Classification {
-
+    @Builder.Default
     private Map<String, Float> probabilities = new HashMap<>();
     private String label;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageServiceResponse.java

@@ -22,8 +22,9 @@ public class ImageServiceResponse {
 
     @JsonProperty(value = "imageMetadata")
     @JsonAlias("data")
+    @Builder.Default
     private List<ImageMetadata> data = new ArrayList<>();
-
+    @Builder.Default
     private List<ImageMetadata> dataCV = new ArrayList<>();
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/ExtractedTable.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExtractedTable {
+
+    private boolean rotated;
+    private ExtractedTableData table;
+    private List<ExtractedTableData> objects;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/ExtractedTableData.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExtractedTableData {
+
+    private String label;
+    private float score;
+    private List<Float> bbox;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableData.java

@@ -15,6 +15,7 @@ import lombok.NoArgsConstructor;
 public class TableData {
 
     private PageInfo pageInfo;
+    @Builder.Default
     private List<TableCells> tableCells = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableExtractorCells.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorCells {
+    private float x0;
+    private float y0;
+    private float x1;
+    private float y1;
+    private float width;
+    private float height;
+    private String label;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableExtractorData.java

@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorData {
+
+    private int page_number;
+    private int image;
+    private List<ExtractedTable> tables;
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableExtractorResponse.java

@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorResponse {
+
+    private String dossierId;
+    private String fileId;
+    private String targetFileExtension;
+    private String responseFileExtension;
+    private String X_TENANT_ID;
+    private List<TableExtractorData> data;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableServiceResponse.java

@@ -19,7 +19,7 @@ public class TableServiceResponse {
     private String operation;
     private String targetFileExtension;
     private String responseFileExtension;
-
+    @Builder.Default
     private List<TableData> data = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/ViewerDocumentService.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
 
+import java.awt.Color;
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
 import java.io.File;
@@ -7,6 +8,8 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.cos.COSName;
@@ -31,6 +34,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.visualization.
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.LayoutGrid;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
 
 import io.micrometer.observation.Observation;
 import io.micrometer.observation.ObservationRegistry;
@@ -54,13 +58,14 @@ public class ViewerDocumentService {
 
     @SneakyThrows
     @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
-    public void createViewerDocument(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {
+    public void createViewerDocument(File originFile, Document document, File destinationFile, Map<Integer, List<TableExtractorCells>> extractedTableCells, boolean layerVisibilityDefaultValue) {
 
         Path tmpFile = Files.createTempFile("tmpViewerDocument", ".pdf");
         PDDocument pdDocument = openPDDocument(originFile);
         LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
 
         PDOptionalContentGroup layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
+        PDOptionalContentGroup tableExtractorLayer = addLayerToDocument(pdDocument, true);
         PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);
 
         for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
@@ -79,7 +84,7 @@ public class ViewerDocumentService {
             // We need to append to the content stream, otherwise the content could be overlapped by following content.
             try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true)) {
 
-                contentStream.beginMarkedContent(COSName.OC, layer);
+                /*contentStream.beginMarkedContent(COSName.OC, layer);
                 contentStream.saveGraphicsState();
 
                 contentStream.setLineWidth(LINE_WIDTH);
@@ -119,6 +124,31 @@ public class ViewerDocumentService {
                     contentStream.endText();
                 }
                 contentStream.restoreGraphicsState();
+                contentStream.endMarkedContent();*/
+
+                contentStream.beginMarkedContent(COSName.OC, tableExtractorLayer);
+                contentStream.saveGraphicsState();
+
+                contentStream.setLineWidth(LINE_WIDTH);
+                for (TableExtractorCells tableCells : extractedTableCells.get(pageNumber)) {
+                    log.info("drawn tableCell {} on page {}",tableCells, pageNumber);
+                    contentStream.setStrokingColor(new Color(0xFF00DD));
+                    contentStream.addRect((float) tableCells.getX0(), (float) tableCells.getY0(), (float) tableCells.getWidth(), (float) tableCells.getHeight());
+                    contentStream.stroke();
+                    contentStream.setFont(font, FONT_SIZE);
+                    contentStream.beginText();
+                    Matrix textMatrix = new Matrix((float) textDeRotationMatrix.getScaleX(),
+                            (float) textDeRotationMatrix.getShearX(),
+                            (float) textDeRotationMatrix.getShearY(),
+                            (float) textDeRotationMatrix.getScaleY(),
+                            tableCells.getX0() ,
+                            tableCells.getY0());
+                    textMatrix.translate(-((font.getStringWidth(tableCells.getLabel()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
+                    contentStream.setTextMatrix(textMatrix);
+                    contentStream.showText(tableCells.getLabel());
+                    contentStream.endText();
+                }
+                contentStream.restoreGraphicsState();
                 contentStream.endMarkedContent();
             }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -26,7 +26,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService, null);
         Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
         var documentFile = new ClassPathResource(fileName).getFile();
-        viewerDocumentService.createViewerDocument(documentFile, document, new File(tmpFileName), true);
+        viewerDocumentService.createViewerDocument(documentFile, document, new File(tmpFileName),null, true);
     }
 
 }