From a1521877d7834bc708e6ea43060b1374ef9768e5 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 23 Feb 2024 12:20:11 +0100 Subject: [PATCH 1/5] RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results added a source label to image properties to enable rules --- .../api/data/redaction/DocumentStructure.java | 5 +- .../api/data/redaction/LayoutEngine.java | 5 ++ .../processor/LayoutParsingPipeline.java | 15 +++-- .../processor/model/graph/nodes/Document.java | 4 ++ .../processor/model/graph/nodes/Footer.java | 4 ++ .../processor/model/graph/nodes/Header.java | 3 + .../processor/model/graph/nodes/Headline.java | 3 + .../processor/model/graph/nodes/Image.java | 4 ++ .../model/graph/nodes/ImageType.java | 2 + .../model/graph/nodes/Paragraph.java | 4 ++ .../processor/model/graph/nodes/Section.java | 3 + .../model/graph/nodes/SemanticNode.java | 8 +++ .../processor/model/graph/nodes/Table.java | 3 + .../model/graph/nodes/TableCell.java | 3 + .../model/image/ClassifiedImage.java | 3 + .../adapter/VisualLayoutParsingAdapter.java | 4 +- .../factory/DocumentGraphFactory.java | 11 +++- .../services/mapper/DocumentDataMapper.java | 1 + .../visualization/LayoutGridService.java | 59 ++++++++++--------- .../service/viewerdoc/ContentStreams.java | 4 +- 20 files changed, 108 insertions(+), 40 deletions(-) create mode 100644 layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java index 01440cb..f569864 100644 --- a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java @@ -3,8 +3,10 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction; import java.awt.geom.Rectangle2D; import java.io.Serializable; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import io.swagger.v3.oas.annotations.media.Schema; @@ -43,7 +45,6 @@ public class DocumentStructure implements Serializable { public static final String ID = "id"; } - @Schema(description = "Object containing the extra field names, a table cell has in its properties field.") public static class TableCellProperties implements Serializable { @@ -115,6 +116,8 @@ public class DocumentStructure implements Serializable { Map properties; @Schema(description = "All child Entries of this Entry.", example = "[1, 2, 3]") List children; + @Schema(description = "Describes the origin of the semantic node",example = "[ALGORITHM]") + Set engines; @Override diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java new file mode 100644 index 0000000..c32cf52 --- /dev/null +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java @@ -0,0 +1,5 @@ +package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction; + +public enum LayoutEngine { + ALGORITHM, AI +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java index 25cf3f8..17800d1 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java @@ -134,6 +134,8 @@ public class LayoutParsingPipeline { layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false); + layoutGridService.addLayoutGrid(viewerDocumentFile,documentGraph,viewerDocumentFile,false,true); + log.info("Storing resulting files for {}", layoutParsingRequest.identifier()); layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph)); @@ -219,11 +221,7 @@ public class LayoutParsingPipeline { addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath())); Map> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse); Map> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse); - Map> signatures = new HashMap<>(); - if(signatures.size() > 0) { - visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse); - } - + Map> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse); ClassificationDocument classificationDocument = new ClassificationDocument(); List classificationPages = new ArrayList<>(); @@ -284,7 +282,12 @@ public class LayoutParsingPipeline { } if(signatures.containsKey(pageNumber)) { - classificationPage.setImages(signatures.get(pageNumber)); + if(classificationPage.getImages() == null ||classificationPage.getImages().size() == 0) { + classificationPage.setImages(signatures.get(pageNumber)); + } + else { + classificationPage.getImages().addAll(signatures.get(pageNumber)); + } } tableExtractionService.extractTables(cleanRulings, classificationPage); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java index c07bffa..8de25ca 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java @@ -10,6 +10,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -30,6 +31,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Document implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); + Set pages; DocumentTree documentTree; Integer numberOfPages; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java index e8e43d1f..14485df 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -26,6 +27,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Footer implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); + List treeId; TextBlock leafTextBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java index 2092c32..9285490 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Header implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; TextBlock leafTextBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java index 95be162..b708c2b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Headline implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; TextBlock leafTextBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java index 1a9c42d..0d9cc26 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -29,6 +30,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Image implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); + List treeId; String id; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java index e863c1a..24cb5c8 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java @@ -6,6 +6,8 @@ public enum ImageType { LOGO, FORMULA, SIGNATURE, + + SIGNATURE_VISUAL, OTHER, OCR; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java index 224b537..71010b6 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -24,6 +25,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Paragraph implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); + List treeId; TextBlock leafTextBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java index 60cc243..ebc39c7 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -27,6 +28,8 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(level = AccessLevel.PRIVATE) public class Section implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; TextBlock textBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java index 811ca45..9afabb8 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java @@ -12,6 +12,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; @@ -21,6 +22,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.textbloc import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock; import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations; +import ch.qos.logback.core.Layout; + public interface SemanticNode { /** @@ -334,6 +337,11 @@ public interface SemanticNode { } } + Set getEngines(); + + default void addEngine(LayoutEngine engine) { + getEngines().add(engine); + } /** * Streams all children located directly underneath this node in the DocumentTree. diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java index 8f77162..f12ee8d 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java @@ -12,6 +12,7 @@ import java.util.Set; import java.util.stream.IntStream; import java.util.stream.Stream; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -31,6 +32,8 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Table implements SemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; DocumentTree documentTree; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java index 3036d94..73ba2d5 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class TableCell implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; int row; int col; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java index 56c36a2..c57fe9a 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java @@ -4,18 +4,21 @@ import java.awt.geom.Rectangle2D; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NonNull; import lombok.RequiredArgsConstructor; @Data @RequiredArgsConstructor +@AllArgsConstructor public class ClassifiedImage { @NonNull private Rectangle2D position; @NonNull private ImageType imageType; + private boolean sourceByAi; private boolean isAppendedToSection; @NonNull private boolean hasTransparency; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java index f91364d..2ddd882 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java @@ -37,7 +37,7 @@ public class VisualLayoutParsingAdapter { public Map> buildExtractedSignaturesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) { Map> signatures = new HashMap<>(); - visualLayoutParsingResponse.getData().forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes()))); + visualLayoutParsingResponse.getData().forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx()+1, tableCell -> new ArrayList<>()).addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes()))); return signatures; } @@ -70,7 +70,7 @@ public class VisualLayoutParsingAdapter { tableObjects.stream().forEach(t -> { if(t.getLabel().equals(SIGNATURES)) { ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(),t.getBox().getY1(),t.getBox().getX2() - t.getBox().getX1(),t.getBox().getY2() - t.getBox().getY1()), - ImageType.SIGNATURE,false,pageNumber); + ImageType.SIGNATURE,true,false,false,pageNumber); signatures.add(signature); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java index 6bde310..3ec26ef 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java @@ -7,12 +7,14 @@ import static java.util.stream.Collectors.toList; import java.awt.geom.Rectangle2D; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter; @@ -95,14 +97,17 @@ public class DocumentGraphFactory { Rectangle2D position = image.getPosition(); Page page = context.getPage(image.getPage()); - Image imageNode = Image.builder() + var imageBuilder = Image.builder() .id(IdBuilder.buildId(Set.of(page), List.of(position))) .imageType(image.getImageType()) .position(position) .transparent(image.isHasTransparency()) .page(page) - .documentTree(context.getDocumentTree()) - .build(); + .documentTree(context.getDocumentTree()); + if(image.isSourceByAi()) { + imageBuilder.engines(new HashSet<>(Set.of(LayoutEngine.AI))); + } + Image imageNode = imageBuilder.build(); page.getMainBody().add(imageNode); List tocId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java index e58a176..4f05c1b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java @@ -80,6 +80,7 @@ public class DocumentDataMapper { .treeId(toPrimitiveIntArray(entry.getTreeId())) .children(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList()) .type(entry.getType()) + .engines(entry.getNode().getEngines()) .atomicBlockIds(atomicTextBlocks) .pageNumbers(entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new)) .properties(properties) diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java index a2ebcc0..b29381b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java @@ -17,6 +17,7 @@ import java.util.stream.Stream; import org.springframework.stereotype.Service; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page; @@ -58,52 +59,56 @@ public class LayoutGridService { static Color HEADER_COLOR = new Color(171, 131, 6); static Color IMAGE_COLOR = new Color(253, 63, 146); + static Color IMAGE_VISUAL_COLOR = new Color(122, 0, 255); @SneakyThrows @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document") public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) { + this.addLayoutGrid(originFile,document,destinationFile,layerVisibilityDefaultValue,false); + } - LayoutGrid layoutGrid = createLayoutGrid(document); + @SneakyThrows + @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document") + public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue, boolean visualParsingGrid) { + + LayoutGrid layoutGrid = createLayoutGrid(document, visualParsingGrid); viewerDocumentService.addVisualizationsOnPage(originFile, destinationFile, Visualizations.builder() - .layer(ContentStreams.KNECON_LAYOUT) + .layer(visualParsingGrid ? ContentStreams.KNECON_VISUAL_PARSING : ContentStreams.KNECON_LAYOUT) .visualizationsOnPages(layoutGrid.getVisualizationsPerPages()) .layerVisibilityDefaultValue(layerVisibilityDefaultValue) .build()); } - - private LayoutGrid createLayoutGrid(Document document) { + private LayoutGrid createLayoutGrid(Document document, boolean visualParsingGrid) { LayoutGrid layoutGrid = new LayoutGrid(document.getNumberOfPages()); - document.streamAllSubNodes().forEach(semanticNode -> { - Color color = switch (semanticNode.getType()) { - case PARAGRAPH -> PARAGRAPH_COLOR; - case TABLE -> TABLE_COLOR; - case SECTION -> SECTION_COLOR; - case HEADLINE -> HEADLINE_COLOR; - case HEADER, FOOTER -> HEADER_COLOR; - case IMAGE -> IMAGE_COLOR; - default -> null; - }; - if (isNotSectionOrTableCellOrDocument(semanticNode)) { - addAsRectangle(semanticNode, layoutGrid, color); - } - if (semanticNode.getType().equals(NodeType.SECTION)) { - addSection(semanticNode, layoutGrid, color); - } - if (semanticNode.getType().equals(NodeType.TABLE)) { - Table table = (Table) semanticNode; - addInnerTableLines(table, layoutGrid); - } - }); - + document.streamAllSubNodes().filter(node -> (node.getEngines().contains(LayoutEngine.AI) && visualParsingGrid ) || (node.getEngines().contains(LayoutEngine.ALGORITHM) && !visualParsingGrid)).forEach(semanticNode -> { + Color color = switch (semanticNode.getType()) { + case PARAGRAPH -> PARAGRAPH_COLOR; + case TABLE -> TABLE_COLOR; + case SECTION -> SECTION_COLOR; + case HEADLINE -> HEADLINE_COLOR; + case HEADER, FOOTER -> HEADER_COLOR; + case IMAGE -> IMAGE_COLOR; + default -> null; + }; + if (isNotSectionOrTableCellOrDocument(semanticNode)) { + addAsRectangle(semanticNode, layoutGrid, color); + } + if (semanticNode.getType().equals(NodeType.SECTION)) { + addSection(semanticNode, layoutGrid, color); + } + if (semanticNode.getType().equals(NodeType.TABLE)) { + Table table = (Table) semanticNode; + addInnerTableLines(table, layoutGrid); + } + }); return layoutGrid; } - private void addInnerTableLines(Table table, LayoutGrid layoutGrid) { if (table.getNumberOfCols() < 1 || table.getNumberOfRows() < 1) { diff --git a/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java b/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java index 882e72b..31d6fca 100644 --- a/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java +++ b/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java @@ -12,6 +12,8 @@ public class ContentStreams { public static Identifier KNECON_LAYOUT = new Identifier("Layout grid", COSName.getPDFName("KNECON_LAYOUT"), true); + public static Identifier KNECON_VISUAL_PARSING = new Identifier("Layout grid - visual", COSName.getPDFName("KNECON_VISUAL_PARSING"), true); + public static Identifier KNECON_OCR = new Identifier("OCR", COSName.getPDFName("KNECON_OCR"), false); public static Identifier KNECON_OCR_TEXT_DEBUG = new Identifier("OCR Text", COSName.getPDFName("KNECON_OCR_TEXT_DEBUG"), true); @@ -24,7 +26,7 @@ public class ContentStreams { public static Identifier ESCAPE_END = new Identifier("escape start", COSName.getPDFName("ESCAPE_END"), false); - public static List allContentStreams = List.of(KNECON_LAYOUT, KNECON_OCR, KNECON_OCR_BBOX_DEBUG, KNECON_OCR_TEXT_DEBUG, OTHER, ESCAPE_START, ESCAPE_END); + public static List allContentStreams = List.of(KNECON_LAYOUT, KNECON_VISUAL_PARSING,KNECON_OCR, KNECON_OCR_BBOX_DEBUG, KNECON_OCR_TEXT_DEBUG, OTHER, ESCAPE_START, ESCAPE_END); public record Identifier(String name, COSName cosName, boolean optionalContent) { From a927cbd9dcb9cb82789c4393ba47f22805081de2 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 23 Feb 2024 12:38:05 +0100 Subject: [PATCH 2/5] RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results fixed tests --- .../python_api/adapter/VisualLayoutParsingAdapter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java index 2ddd882..398d5de 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java @@ -37,9 +37,13 @@ public class VisualLayoutParsingAdapter { public Map> buildExtractedSignaturesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) { Map> signatures = new HashMap<>(); - visualLayoutParsingResponse.getData().forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx()+1, tableCell -> new ArrayList<>()).addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes()))); + if(visualLayoutParsingResponse.getData() != null) { + visualLayoutParsingResponse.getData() + .forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx() + 1, tableCell -> new ArrayList<>()) + .addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes()))); + } + return signatures; - return signatures; } From 71477dabde5d797f1499bc08d9d448177d65c695 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 23 Feb 2024 12:46:51 +0100 Subject: [PATCH 3/5] RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results codestyle --- .../api/data/redaction/DocumentStructure.java | 4 +- .../visualization/LayoutGridService.java | 51 +++++++++++-------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java index f569864..523308c 100644 --- a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java @@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction; import java.awt.geom.Rectangle2D; import java.io.Serializable; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -45,6 +44,7 @@ public class DocumentStructure implements Serializable { public static final String ID = "id"; } + @Schema(description = "Object containing the extra field names, a table cell has in its properties field.") public static class TableCellProperties implements Serializable { @@ -116,7 +116,7 @@ public class DocumentStructure implements Serializable { Map properties; @Schema(description = "All child Entries of this Entry.", example = "[1, 2, 3]") List children; - @Schema(description = "Describes the origin of the semantic node",example = "[ALGORITHM]") + @Schema(description = "Describes the origin of the semantic node", example = "[ALGORITHM]") Set engines; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java index b29381b..36ffaa1 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java @@ -61,12 +61,15 @@ public class LayoutGridService { static Color IMAGE_VISUAL_COLOR = new Color(122, 0, 255); + @SneakyThrows @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document") public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) { - this.addLayoutGrid(originFile,document,destinationFile,layerVisibilityDefaultValue,false); + + this.addLayoutGrid(originFile, document, destinationFile, layerVisibilityDefaultValue, false); } + @SneakyThrows @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document") public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue, boolean visualParsingGrid) { @@ -82,33 +85,37 @@ public class LayoutGridService { .build()); } + private LayoutGrid createLayoutGrid(Document document, boolean visualParsingGrid) { LayoutGrid layoutGrid = new LayoutGrid(document.getNumberOfPages()); - document.streamAllSubNodes().filter(node -> (node.getEngines().contains(LayoutEngine.AI) && visualParsingGrid ) || (node.getEngines().contains(LayoutEngine.ALGORITHM) && !visualParsingGrid)).forEach(semanticNode -> { - Color color = switch (semanticNode.getType()) { - case PARAGRAPH -> PARAGRAPH_COLOR; - case TABLE -> TABLE_COLOR; - case SECTION -> SECTION_COLOR; - case HEADLINE -> HEADLINE_COLOR; - case HEADER, FOOTER -> HEADER_COLOR; - case IMAGE -> IMAGE_COLOR; - default -> null; - }; - if (isNotSectionOrTableCellOrDocument(semanticNode)) { - addAsRectangle(semanticNode, layoutGrid, color); - } - if (semanticNode.getType().equals(NodeType.SECTION)) { - addSection(semanticNode, layoutGrid, color); - } - if (semanticNode.getType().equals(NodeType.TABLE)) { - Table table = (Table) semanticNode; - addInnerTableLines(table, layoutGrid); - } - }); + document.streamAllSubNodes() + .filter(node -> (node.getEngines().contains(LayoutEngine.AI) && visualParsingGrid) || (node.getEngines().contains(LayoutEngine.ALGORITHM) && !visualParsingGrid)) + .forEach(semanticNode -> { + Color color = switch (semanticNode.getType()) { + case PARAGRAPH -> PARAGRAPH_COLOR; + case TABLE -> TABLE_COLOR; + case SECTION -> SECTION_COLOR; + case HEADLINE -> HEADLINE_COLOR; + case HEADER, FOOTER -> HEADER_COLOR; + case IMAGE -> IMAGE_COLOR; + default -> null; + }; + if (isNotSectionOrTableCellOrDocument(semanticNode)) { + addAsRectangle(semanticNode, layoutGrid, color); + } + if (semanticNode.getType().equals(NodeType.SECTION)) { + addSection(semanticNode, layoutGrid, color); + } + if (semanticNode.getType().equals(NodeType.TABLE)) { + Table table = (Table) semanticNode; + addInnerTableLines(table, layoutGrid); + } + }); return layoutGrid; } + private void addInnerTableLines(Table table, LayoutGrid layoutGrid) { if (table.getNumberOfCols() < 1 || table.getNumberOfRows() < 1) { From 2c171b6a9e63c924ed685f2702fc7d971786ac83 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 23 Feb 2024 13:55:11 +0100 Subject: [PATCH 4/5] RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results codestyle --- .../api/data/redaction/LayoutEngine.java | 3 +- .../processor/LayoutParsingPipeline.java | 17 +++-- .../processor/model/graph/nodes/Document.java | 13 +++- .../processor/model/graph/nodes/Image.java | 4 +- .../processor/model/graph/nodes/Section.java | 7 +- .../model/graph/nodes/SemanticNode.java | 15 +++- .../processor/model/graph/nodes/Table.java | 74 +++++++++++++------ .../adapter/VisualLayoutParsingAdapter.java | 13 +++- .../factory/DocumentGraphFactory.java | 6 +- .../service/viewerdoc/ContentStreams.java | 9 ++- 10 files changed, 113 insertions(+), 48 deletions(-) diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java index c32cf52..d5f541e 100644 --- a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java @@ -1,5 +1,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction; public enum LayoutEngine { - ALGORITHM, AI + ALGORITHM, + AI } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java index 17800d1..772f2f4 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java @@ -101,8 +101,10 @@ public class LayoutParsingPipeline { .orElse(originFile); VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse(); - if (layoutParsingRequest.visualLayoutParsingFileId().isPresent()) { - visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId().get()); + if (layoutParsingRequest.visualLayoutParsingFileId() + .isPresent()) { + visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId() + .get()); } ImageServiceResponse imageServiceResponse = new ImageServiceResponse(); @@ -124,7 +126,7 @@ public class LayoutParsingPipeline { imageServiceResponse, tableServiceResponse, visualLayoutParsingResponse, - layoutParsingRequest.identifier().toString()); + layoutParsingRequest.identifier().toString()); log.info("Building document graph for {}", layoutParsingRequest.identifier()); @@ -134,7 +136,7 @@ public class LayoutParsingPipeline { layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false); - layoutGridService.addLayoutGrid(viewerDocumentFile,documentGraph,viewerDocumentFile,false,true); + layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, true); log.info("Storing resulting files for {}", layoutParsingRequest.identifier()); @@ -281,11 +283,10 @@ public class LayoutParsingPipeline { imageServiceResponseAdapter.findOcr(classificationPage); } - if(signatures.containsKey(pageNumber)) { - if(classificationPage.getImages() == null ||classificationPage.getImages().size() == 0) { + if (signatures.containsKey(pageNumber)) { + if (classificationPage.getImages() == null || classificationPage.getImages().size() == 0) { classificationPage.setImages(signatures.get(pageNumber)); - } - else { + } else { classificationPage.getImages().addAll(signatures.get(pageNumber)); } } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java index 8de25ca..23a3cd0 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java @@ -60,13 +60,15 @@ public class Document implements GenericSemanticNode { public List
getMainSections() { - return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node).collect(Collectors.toList()); + return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node) + .collect(Collectors.toList()); } public Stream streamTerminalTextBlocksInOrder() { - return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock); + return streamAllNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock); } @@ -87,13 +89,16 @@ public class Document implements GenericSemanticNode { @Override public Headline getHeadline() { - return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build()); + return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node) + .findFirst() + .orElse(Headline.builder().build()); } private Stream streamAllNodes() { - return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode); + return documentTree.allEntriesInOrder() + .map(DocumentTree.Entry::getNode); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java index 0d9cc26..bcfb039 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java @@ -70,7 +70,9 @@ public class Image implements GenericSemanticNode { @Override public TextBlock getTextBlock() { - return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + return streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java index ebc39c7..6680c01 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java @@ -53,7 +53,8 @@ public class Section implements GenericSemanticNode { public boolean hasTables() { - return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent(); + return streamAllSubNodesOfType(NodeType.TABLE).findAny() + .isPresent(); } @@ -61,7 +62,9 @@ public class Section implements GenericSemanticNode { public TextBlock getTextBlock() { if (textBlock == null) { - textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } return textBlock; } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java index 9afabb8..11a612c 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java @@ -22,8 +22,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.textbloc import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock; import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations; -import ch.qos.logback.core.Layout; - public interface SemanticNode { /** @@ -337,12 +335,24 @@ public interface SemanticNode { } } + + /** + * returns the set of layoutengines + * + * @return set of layoutengines + */ Set getEngines(); + + /** + * adds a layoutengine to the set + */ default void addEngine(LayoutEngine engine) { + getEngines().add(engine); } + /** * Streams all children located directly underneath this node in the DocumentTree. * @@ -434,6 +444,7 @@ public interface SemanticNode { /** * TODO: this produces unwanted results for sections spanning multiple columns. * Computes the Union of the bounding boxes of all children recursively. + * * @return The union of the BoundingBoxes of all children */ private Map getBBoxFromChildren() { diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java index f12ee8d..5bed37b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java @@ -48,6 +48,7 @@ public class Table implements SemanticNode { @EqualsAndHashCode.Exclude Map bBoxCache; + /** * Streams all entities in this table, that appear in a row, which contains any of the provided strings. * @@ -56,8 +57,7 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowContainsStringsIgnoreCase(List strings) { - return IntStream.range(0, numberOfRows) - .boxed() + return IntStream.range(0, numberOfRows).boxed() .filter(row -> rowContainsStringsIgnoreCase(row, strings)) .flatMap(this::streamRow) .map(TableCell::getEntities) @@ -74,8 +74,11 @@ public class Table implements SemanticNode { */ public boolean rowContainsStringsIgnoreCase(Integer row, List strings) { - String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT); - return strings.stream().map(String::toLowerCase).allMatch(rowText::contains); + String rowText = streamRow(row).map(TableCell::getTextBlock) + .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT); + return strings.stream() + .map(String::toLowerCase) + .allMatch(rowText::contains); } @@ -88,9 +91,13 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowHasHeaderAndValue(String header, String value) { - List vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList(); + List vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)) + .map(TableCell::getCol) + .toList(); return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream() - .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream); + .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))) + .map(TableCell::getEntities) + .flatMap(Collection::stream); } @@ -103,9 +110,13 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List values) { - List colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList(); + List colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)) + .map(TableCell::getCol) + .toList(); return streamTableCells().filter(tableCellNode -> colsWithHeader.stream() - .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream); + .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))) + .map(TableCell::getEntities) + .flatMap(Collection::stream); } @@ -117,12 +128,15 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowContainsEntitiesOfType(List types) { - List rowsWithEntityOfType = IntStream.range(0, numberOfRows) - .boxed() - .filter(rowNumber -> streamEntityTypesInRow(rowNumber).anyMatch(existingType -> types.stream().anyMatch(typeToCheck -> typeToCheck.equals(existingType)))) + List rowsWithEntityOfType = IntStream.range(0, numberOfRows).boxed() + .filter(rowNumber -> streamEntityTypesInRow(rowNumber).anyMatch(existingType -> types.stream() + .anyMatch(typeToCheck -> typeToCheck.equals(existingType)))) .toList(); - return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream); + return rowsWithEntityOfType.stream() + .flatMap(this::streamRow) + .map(TableCell::getEntities) + .flatMap(Collection::stream); } @@ -134,18 +148,24 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowContainsNoEntitiesOfType(List types) { - List rowsWithNoEntityOfType = IntStream.range(0, numberOfRows) - .boxed() - .filter(rowNumber -> streamEntityTypesInRow(rowNumber).noneMatch(existingType -> types.stream().anyMatch(typeToCheck -> typeToCheck.equals(existingType)))) + List rowsWithNoEntityOfType = IntStream.range(0, numberOfRows).boxed() + .filter(rowNumber -> streamEntityTypesInRow(rowNumber).noneMatch(existingType -> types.stream() + .anyMatch(typeToCheck -> typeToCheck.equals(existingType)))) .toList(); - return rowsWithNoEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream); + return rowsWithNoEntityOfType.stream() + .flatMap(this::streamRow) + .map(TableCell::getEntities) + .flatMap(Collection::stream); } private Stream streamEntityTypesInRow(Integer rowNumber) { - return streamRow(rowNumber).map(TableCell::getEntities).flatMap(Collection::stream).map(RedactionEntity::getType).distinct(); + return streamRow(rowNumber).map(TableCell::getEntities) + .flatMap(Collection::stream) + .map(RedactionEntity::getType) + .distinct(); } @@ -162,7 +182,8 @@ public class Table implements SemanticNode { throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols)); } int idx = row * numberOfCols + col; - return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode(); + return (TableCell) documentTree.getEntryById(treeId).getChildren() + .get(idx).getNode(); } @@ -199,7 +220,8 @@ public class Table implements SemanticNode { */ public Stream streamCol(int col) { - return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col)); + return IntStream.range(0, numberOfRows).boxed() + .map(row -> getCell(row, col)); } @@ -211,9 +233,11 @@ public class Table implements SemanticNode { */ public Stream streamRow(int row) { - return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col)); + return IntStream.range(0, numberOfCols).boxed() + .map(col -> getCell(row, col)); } + /** * Streams all TableCells row-wise and filters them with header == true. * @@ -234,7 +258,8 @@ public class Table implements SemanticNode { */ public Stream streamHeadersForCell(int row, int col) { - return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader); + return Stream.concat(streamRow(row), streamCol(col)) + .filter(TableCell::isHeader); } @@ -307,7 +332,9 @@ public class Table implements SemanticNode { public TextBlock getTextBlock() { if (textBlock == null) { - textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } return textBlock; } @@ -318,6 +345,8 @@ public class Table implements SemanticNode { return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary(); } + + @Override public Map getBBox() { @@ -326,4 +355,5 @@ public class Table implements SemanticNode { } return bBoxCache; } + } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java index 398d5de..6fe668b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java @@ -24,6 +24,7 @@ public class VisualLayoutParsingAdapter { private static String SIGNATURES = "signature"; + public Map> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) { Map> tableCells = new HashMap<>(); @@ -34,10 +35,11 @@ public class VisualLayoutParsingAdapter { } + public Map> buildExtractedSignaturesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) { Map> signatures = new HashMap<>(); - if(visualLayoutParsingResponse.getData() != null) { + if (visualLayoutParsingResponse.getData() != null) { visualLayoutParsingResponse.getData() .forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx() + 1, tableCell -> new ArrayList<>()) .addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes()))); @@ -67,14 +69,17 @@ public class VisualLayoutParsingAdapter { } + public List convertSignatures(int pageNumber, List tableObjects) { List signatures = new ArrayList<>(); tableObjects.stream().forEach(t -> { - if(t.getLabel().equals(SIGNATURES)) { - ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(),t.getBox().getY1(),t.getBox().getX2() - t.getBox().getX1(),t.getBox().getY2() - t.getBox().getY1()), - ImageType.SIGNATURE,true,false,false,pageNumber); + if (t.getLabel().equals(SIGNATURES)) { + ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(), + t.getBox().getY1(), + t.getBox().getX2() - t.getBox().getX1(), + t.getBox().getY2() - t.getBox().getY1()), ImageType.SIGNATURE, true, false, false, pageNumber); signatures.add(signature); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java index 3ec26ef..778ad91 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java @@ -20,8 +20,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage; -import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage; -import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer; @@ -33,6 +31,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section; import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock; +import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage; +import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock; import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder; import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations; @@ -104,7 +104,7 @@ public class DocumentGraphFactory { .transparent(image.isHasTransparency()) .page(page) .documentTree(context.getDocumentTree()); - if(image.isSourceByAi()) { + if (image.isSourceByAi()) { imageBuilder.engines(new HashSet<>(Set.of(LayoutEngine.AI))); } Image imageNode = imageBuilder.build(); diff --git a/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java b/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java index 31d6fca..937f75d 100644 --- a/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java +++ b/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java @@ -26,7 +26,14 @@ public class ContentStreams { public static Identifier ESCAPE_END = new Identifier("escape start", COSName.getPDFName("ESCAPE_END"), false); - public static List allContentStreams = List.of(KNECON_LAYOUT, KNECON_VISUAL_PARSING,KNECON_OCR, KNECON_OCR_BBOX_DEBUG, KNECON_OCR_TEXT_DEBUG, OTHER, ESCAPE_START, ESCAPE_END); + public static List allContentStreams = List.of(KNECON_LAYOUT, + KNECON_VISUAL_PARSING, + KNECON_OCR, + KNECON_OCR_BBOX_DEBUG, + KNECON_OCR_TEXT_DEBUG, + OTHER, + ESCAPE_START, + ESCAPE_END); public record Identifier(String name, COSName cosName, boolean optionalContent) { From 477f6af886318fbde2bebe14b6ad4eeb42715984 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 23 Feb 2024 14:02:53 +0100 Subject: [PATCH 5/5] RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results checkstyle --- .../processor/model/graph/nodes/SemanticNode.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java index 11a612c..e35a83e 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java @@ -337,15 +337,15 @@ public interface SemanticNode { /** - * returns the set of layoutengines + * returns the set of layoutengines. * - * @return set of layoutengines + * @return set of layoutengines. */ Set getEngines(); /** - * adds a layoutengine to the set + * adds a layoutengine to the set. */ default void addEngine(LayoutEngine engine) {