From 0f11d901f649284bf787182b0fd67d9566e22752 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 21 Dec 2023 08:54:03 +0100 Subject: [PATCH] RED-7964: Prototype Visual Layout Parsing: added layer for visual layout parsing results (true by default) added label to drawn boxes --- .../processor/LayoutParsingPipeline.java | 3 +- .../adapter/VisualLayoutParsingAdapter.java | 28 ++++++++-------- .../table/VisualLayoutParsingBoxValue.java | 19 +++++++++++ .../table/VisualLayoutParsingResult.java | 22 +++++++++++++ .../visualization/ViewerDocumentService.java | 33 +++++++++++++++---- 5 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBoxValue.java create mode 100644 layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResult.java diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java index 10a66db..02e0bab 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java @@ -35,6 +35,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells; import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse; import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse; +import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult; import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService; import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService; import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService; @@ -108,7 +109,7 @@ public class LayoutParsingPipeline { layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph)); layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph)); - Map> extractedTableCells = visualLayoutParsingAdapter.buildExtractedTablesPerPage(visualLayoutParsingResponse); + Map> extractedTableCells = visualLayoutParsingAdapter.buildExtractedTablesPerPage(visualLayoutParsingResponse); try (var out = new ByteArrayOutputStream()) { viewerDocumentService.createViewerDocument(originDocument, documentGraph, out, extractedTableCells, false); layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java index 843fdbe..de378d8 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java @@ -10,6 +10,7 @@ import org.springframework.stereotype.Service; import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells; import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingBox; import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse; +import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult; import lombok.RequiredArgsConstructor; @@ -17,9 +18,9 @@ import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public class VisualLayoutParsingAdapter { - public Map> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) { + public Map> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) { - Map> tableCells = new HashMap<>(); + Map> tableCells = new HashMap<>(); visualLayoutParsingResponse.getData() .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertTableCells(tableData.getBoxes()))); @@ -28,21 +29,20 @@ public class VisualLayoutParsingAdapter { } - public List convertTableCells(List tableObjects) { + public List convertTableCells(List tableObjects) { - List parsedTableCells = new ArrayList<>(); + List parsedTableCells = new ArrayList<>(); tableObjects.stream().forEach(t -> { - if(t.getProbability() > 0.9) { - TableCells tableCells = new TableCells(); - tableCells.setX0(t.getBox().getX1()); - tableCells.setX1(t.getBox().getX2()); - tableCells.setY0(t.getBox().getY1()); - tableCells.setY1(t.getBox().getY2()); - tableCells.setWidth(tableCells.getX1() - tableCells.getX0()); - tableCells.setHeight(tableCells.getY1() - tableCells.getY0()); - parsedTableCells.add(tableCells); - } + VisualLayoutParsingResult result = new VisualLayoutParsingResult(); + result.setX0(t.getBox().getX1()); + result.setX1(t.getBox().getX2()); + result.setY0(t.getBox().getY1()); + result.setY1(t.getBox().getY2()); + result.setWidth(result.getX1() - result.getX0()); + result.setHeight(result.getY1() - result.getY0()); + result.setLabel(t.getLabel()); + parsedTableCells.add(result); }); return parsedTableCells; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBoxValue.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBoxValue.java new file mode 100644 index 0000000..2961894 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBoxValue.java @@ -0,0 +1,19 @@ +package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class VisualLayoutParsingBoxValue { + + private float x1; + private float y1; + private float x2; + private float y2; + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResult.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResult.java new file mode 100644 index 0000000..643cf2b --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResult.java @@ -0,0 +1,22 @@ +package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class VisualLayoutParsingResult { + + private float x0; + private float y0; + private float x1; + private float y1; + private float width; + private float height; + private String label; + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/ViewerDocumentService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/ViewerDocumentService.java index 713cba3..fb6e9e8 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/ViewerDocumentService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/ViewerDocumentService.java @@ -34,6 +34,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.visualization. import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText; import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage; import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells; +import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -55,7 +56,7 @@ public class ViewerDocumentService { public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream, - Map> extractedTableCells, + Map> extractedTableCells, boolean layerVisibilityDefaultValue) { LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document); @@ -63,6 +64,7 @@ public class ViewerDocumentService { // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast. Set dictionariesToUpdate = new HashSet<>(); PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue); + PDOptionalContentGroup visualLayoutParsingLayer = addLayerToDocument(pdDocument, dictionariesToUpdate, true); PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA); for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) { @@ -105,11 +107,6 @@ public class ViewerDocumentService { contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight()); contentStream.fill(); } - for (TableCells tableCells : extractedTableCells.get(pageNumber)) { - contentStream.setStrokingColor(Color.CYAN); - contentStream.addRect((float) tableCells.getX0(), (float) tableCells.getY0(), (float) tableCells.getWidth(), (float) tableCells.getHeight()); - contentStream.stroke(); - } for (PlacedText placedText : visualizationsOnPage.getPlacedTexts()) { contentStream.setFont(font, FONT_SIZE); contentStream.beginText(); @@ -126,6 +123,30 @@ public class ViewerDocumentService { } contentStream.restoreGraphicsState(); contentStream.endMarkedContent(); + + contentStream.beginMarkedContent(COSName.OC, visualLayoutParsingLayer); + contentStream.saveGraphicsState(); + + contentStream.setLineWidth(LINE_WIDTH); + for (VisualLayoutParsingResult tableCells : extractedTableCells.get(pageNumber)) { + contentStream.setStrokingColor(new Color(0xFF0000)); + contentStream.addRect((float) tableCells.getX0(), (float) tableCells.getY0(), (float) tableCells.getWidth(), (float) tableCells.getHeight()); + contentStream.stroke(); + contentStream.setFont(font, FONT_SIZE); + contentStream.beginText(); + Matrix textMatrix = new Matrix((float) textDeRotationMatrix.getScaleX(), + (float) textDeRotationMatrix.getShearX(), + (float) textDeRotationMatrix.getShearY(), + (float) textDeRotationMatrix.getScaleY(), + tableCells.getX0() , + tableCells.getY0()); + textMatrix.translate(-((font.getStringWidth(tableCells.getLabel()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE); + contentStream.setTextMatrix(textMatrix); + contentStream.showText(tableCells.getLabel()); + contentStream.endText(); + } + contentStream.restoreGraphicsState(); + contentStream.endMarkedContent(); } dictionariesToUpdate.add(pdPage.getCOSObject()); dictionariesToUpdate.add(pdPage.getResources().getCOSObject());