From a5fcebce30479b2a945b15385b6661ef43a980a0 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 17 May 2024 07:34:05 +0200 Subject: [PATCH 1/4] RED-3813: Recategorize same image as experimental feature added representation to image and DocumentStructure --- .../internal/api/data/redaction/DocumentStructure.java | 2 ++ .../service/layoutparser/processor/LayoutParsingPipeline.java | 2 +- .../layoutparser/processor/model/graph/nodes/Image.java | 2 ++ .../layoutparser/processor/model/image/ClassifiedImage.java | 2 ++ .../python_api/adapter/ImageServiceResponseAdapter.java | 4 ++-- .../processor/python_api/model/image/ImageMetadata.java | 1 + 6 files changed, 10 insertions(+), 3 deletions(-) diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java index 4b26f52..c349ab5 100644 --- a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java @@ -43,6 +43,8 @@ public class DocumentStructure implements Serializable { public static final String POSITION = "position"; public static final String ID = "id"; + public static final String REPRESENTATION_HASH="representationHash"; + } @Schema(description = "Object containing the extra field names, a table cell has in its properties field.") diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java index 48e230e..e095e62 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java @@ -281,7 +281,7 @@ public class LayoutParsingPipeline { pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>()) .addAll(graphics.stream() - .map(g -> new ClassifiedImage(new Rectangle2D.Double(g.x1, g.y1, g.width(), g.height()), ImageType.GRAPHIC, false, stripper.getPageNumber())) + .map(g -> new ClassifiedImage(new Rectangle2D.Double(g.x1, g.y1, g.width(), g.height()), ImageType.GRAPHIC, false, stripper.getPageNumber(),"")) .toList()); ClassificationPage classificationPage = switch (layoutParsingType) { diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java index 0abb37d..7a521dd 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java @@ -35,6 +35,8 @@ public class Image implements GenericSemanticNode { List treeId; String id; + String representationHash; + ImageType imageType; boolean transparent; Rectangle2D position; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java index c57fe9a..40931be 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java @@ -24,5 +24,7 @@ public class ClassifiedImage { private boolean hasTransparency; @NonNull private int page; + @NonNull + private String representation; } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/ImageServiceResponseAdapter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/ImageServiceResponseAdapter.java index 968bfbd..7b6c840 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/ImageServiceResponseAdapter.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/ImageServiceResponseAdapter.java @@ -32,7 +32,7 @@ public class ImageServiceResponseAdapter { .add(new ClassifiedImage(new Rectangle2D.Double(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), - imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber())); + imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber(),imageMetadata.getRepresentation())); }); // Currently This is a copy but, it will be changed later because i don' t think that we should unclassified images. @@ -44,7 +44,7 @@ public class ImageServiceResponseAdapter { .add(new ClassifiedImage(new Rectangle2D.Double(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), - imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber())); + imageMetadata.getGeometry().getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber(),imageMetadata.getRepresentation())); }); return images; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageMetadata.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageMetadata.java index d0c9107..d2cba90 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageMetadata.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageMetadata.java @@ -12,6 +12,7 @@ import lombok.NoArgsConstructor; public class ImageMetadata { private Classification classification; + private String representation; private Position position; private Geometry geometry; private Filters filters; From 9be672c7288a083242c2950f8fe9027011ad6182 Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 28 May 2024 13:51:45 +0200 Subject: [PATCH 2/4] RED-3813: Recategorize same image as experimental feature working on pushing properties to persistence service --- .../api/data/redaction/DocumentStructure.java | 29 +++++++++++++++---- .../services/mapper/PropertiesMapper.java | 1 + 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java index c349ab5..c7a8499 100644 --- a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java @@ -43,7 +43,7 @@ public class DocumentStructure implements Serializable { public static final String POSITION = "position"; public static final String ID = "id"; - public static final String REPRESENTATION_HASH="representationHash"; + public static final String REPRESENTATION_HASH = "representationHash"; } @@ -69,11 +69,25 @@ public class DocumentStructure implements Serializable { public static Rectangle2D parseRectangle2D(String bBox) { - List floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList(); + List floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)) + .map(Float::parseFloat) + .toList(); return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3)); } + public static double[] parseRepresentationVector(String representationHash) { + + String[] stringArray = representationHash.split("[,\\s]+"); + double[] doubleArray = new double[stringArray.length]; + for (int i = 0; i < stringArray.length; i++) { + doubleArray[i] = Double.parseDouble(stringArray[i]); + } + + return doubleArray; + } + + public EntryData get(List tocId) { if (tocId.isEmpty()) { @@ -89,19 +103,24 @@ public class DocumentStructure implements Serializable { public Stream streamAllEntries() { - return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentStructure::flatten); + return Stream.concat(Stream.of(root), root.children.stream()) + .flatMap(DocumentStructure::flatten); } public String toString() { - return String.join("\n", streamAllEntries().map(EntryData::toString).toList()); + return String.join("\n", + streamAllEntries().map(EntryData::toString) + .toList()); } private static Stream flatten(EntryData entry) { - return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentStructure::flatten)); + return Stream.concat(Stream.of(entry), + entry.children.stream() + .flatMap(DocumentStructure::flatten)); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java index f4ebbd5..453838a 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java @@ -24,6 +24,7 @@ public class PropertiesMapper { properties.put(DocumentStructure.ImageProperties.TRANSPARENT, String.valueOf(image.isTransparent())); properties.put(DocumentStructure.ImageProperties.POSITION, toString(image.getPosition())); properties.put(DocumentStructure.ImageProperties.ID, image.getId()); + properties.put(DocumentStructure.ImageProperties.REPRESENTATION_HASH, image.getRepresentationHash()); return properties; } From 9ecf9ca19f851d3773773ed0cd7cce6ae508703c Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 5 Jun 2024 14:20:33 +0200 Subject: [PATCH 3/4] RED-3813: Recategorize same image as experimental feature now writing hash into structure --- .../processor/LayoutParsingPipeline.java | 36 +++++++++++++------ .../adapter/VisualLayoutParsingAdapter.java | 2 +- .../factory/DocumentGraphFactory.java | 3 ++ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java index e095e62..f5c333c 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java @@ -105,21 +105,28 @@ public class LayoutParsingPipeline { log.info("Starting layout parsing for {}", layoutParsingRequest.identifier()); File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId()); - File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile); + File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()) + .orElse(originFile); VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse(); - if (layoutParsingRequest.visualLayoutParsingFileId().isPresent()) { - visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId().get()); + if (layoutParsingRequest.visualLayoutParsingFileId() + .isPresent()) { + visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId() + .get()); } ImageServiceResponse imageServiceResponse = new ImageServiceResponse(); - if (layoutParsingRequest.imagesFileStorageId().isPresent()) { - imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get()); + if (layoutParsingRequest.imagesFileStorageId() + .isPresent()) { + imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId() + .get()); } TableServiceResponse tableServiceResponse = new TableServiceResponse(); - if (layoutParsingRequest.tablesFileStorageId().isPresent()) { - tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get()); + if (layoutParsingRequest.tablesFileStorageId() + .isPresent()) { + tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId() + .get()); } ClassificationDocument classificationDocument = parseLayout(settings.getLayoutParsingTypeOverride() == null // @@ -130,14 +137,17 @@ public class LayoutParsingPipeline { visualLayoutParsingResponse, layoutParsingRequest.identifier()); - log.info("Building document graph for {}", layoutParsingRequest.identifier()); - Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null // ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument); log.info("Creating viewer document for {}", layoutParsingRequest.identifier()); - layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, layoutParsingRequest.visualLayoutParsingFileId().isPresent()); + layoutGridService.addLayoutGrid(viewerDocumentFile, + documentGraph, + viewerDocumentFile, + false, + layoutParsingRequest.visualLayoutParsingFileId() + .isPresent()); log.info("Storing resulting files for {}", layoutParsingRequest.identifier()); @@ -281,7 +291,11 @@ public class LayoutParsingPipeline { pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>()) .addAll(graphics.stream() - .map(g -> new ClassifiedImage(new Rectangle2D.Double(g.x1, g.y1, g.width(), g.height()), ImageType.GRAPHIC, false, stripper.getPageNumber(),"")) + .map(g -> new ClassifiedImage(new Rectangle2D.Double(g.x1, g.y1, g.width(), g.height()), + ImageType.GRAPHIC, + false, + stripper.getPageNumber(), + "")) .toList()); ClassificationPage classificationPage = switch (layoutParsingType) { diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java index 9972310..c5cb41b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java @@ -79,7 +79,7 @@ public class VisualLayoutParsingAdapter { ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(), t.getBox().getY1(), t.getBox().getX2() - t.getBox().getX1(), - t.getBox().getY2() - t.getBox().getY1()), ImageType.SIGNATURE, true, false, false, pageNumber); + t.getBox().getY2() - t.getBox().getY1()), ImageType.SIGNATURE, true, false, false, pageNumber,""); signatures.add(signature); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java index 36ee3eb..26ef65c 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java @@ -43,8 +43,10 @@ import lombok.Builder; import lombok.Getter; import lombok.experimental.FieldDefaults; import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; @UtilityClass +@Slf4j public class DocumentGraphFactory { public Document buildDocumentGraph(LayoutParsingType layoutParsingType, ClassificationDocument document) { @@ -138,6 +140,7 @@ public class DocumentGraphFactory { .position(position) .transparent(image.isHasTransparency()) .page(page) + .representationHash(image.getRepresentation()) .documentTree(context.getDocumentTree()) .build(); page.getMainBody().add(imageNode); From 5c2844fe3110c408c5f9157cd0df0863eb37e72f Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 26 Jun 2024 09:08:37 +0200 Subject: [PATCH 4/4] RED-3813: Recategorize same image as experimental feature fixed failing test --- .../server/segmentation/PdfSegmentationServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java index 1981530..38cd056 100644 --- a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java +++ b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java @@ -187,7 +187,7 @@ public class PdfSegmentationServiceTest extends AbstractTest { imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)), imageMetadata.isAlpha(), - imageMetadata.getPosition().getPageNumber()))); + imageMetadata.getPosition().getPageNumber(), ""))); System.out.println("object"); }