diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java index 01440cb..523308c 100644 --- a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java @@ -5,6 +5,7 @@ import java.io.Serializable; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import io.swagger.v3.oas.annotations.media.Schema; @@ -115,6 +116,8 @@ public class DocumentStructure implements Serializable { Map properties; @Schema(description = "All child Entries of this Entry.", example = "[1, 2, 3]") List children; + @Schema(description = "Describes the origin of the semantic node", example = "[ALGORITHM]") + Set engines; @Override diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java new file mode 100644 index 0000000..d5f541e --- /dev/null +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java @@ -0,0 +1,6 @@ +package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction; + +public enum LayoutEngine { + ALGORITHM, + AI +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java index 25cf3f8..772f2f4 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java @@ -101,8 +101,10 @@ public class LayoutParsingPipeline { .orElse(originFile); VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse(); - if (layoutParsingRequest.visualLayoutParsingFileId().isPresent()) { - visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId().get()); + if (layoutParsingRequest.visualLayoutParsingFileId() + .isPresent()) { + visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId() + .get()); } ImageServiceResponse imageServiceResponse = new ImageServiceResponse(); @@ -124,7 +126,7 @@ public class LayoutParsingPipeline { imageServiceResponse, tableServiceResponse, visualLayoutParsingResponse, - layoutParsingRequest.identifier().toString()); + layoutParsingRequest.identifier().toString()); log.info("Building document graph for {}", layoutParsingRequest.identifier()); @@ -134,6 +136,8 @@ public class LayoutParsingPipeline { layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false); + layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, true); + log.info("Storing resulting files for {}", layoutParsingRequest.identifier()); layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph)); @@ -219,11 +223,7 @@ public class LayoutParsingPipeline { addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath())); Map> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse); Map> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse); - Map> signatures = new HashMap<>(); - if(signatures.size() > 0) { - visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse); - } - + Map> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse); ClassificationDocument classificationDocument = new ClassificationDocument(); List classificationPages = new ArrayList<>(); @@ -283,8 +283,12 @@ public class LayoutParsingPipeline { imageServiceResponseAdapter.findOcr(classificationPage); } - if(signatures.containsKey(pageNumber)) { - classificationPage.setImages(signatures.get(pageNumber)); + if (signatures.containsKey(pageNumber)) { + if (classificationPage.getImages() == null || classificationPage.getImages().size() == 0) { + classificationPage.setImages(signatures.get(pageNumber)); + } else { + classificationPage.getImages().addAll(signatures.get(pageNumber)); + } } tableExtractionService.extractTables(cleanRulings, classificationPage); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java index c07bffa..23a3cd0 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java @@ -10,6 +10,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -30,6 +31,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Document implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); + Set pages; DocumentTree documentTree; Integer numberOfPages; @@ -56,13 +60,15 @@ public class Document implements GenericSemanticNode { public List
getMainSections() { - return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node).collect(Collectors.toList()); + return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node) + .collect(Collectors.toList()); } public Stream streamTerminalTextBlocksInOrder() { - return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock); + return streamAllNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock); } @@ -83,13 +89,16 @@ public class Document implements GenericSemanticNode { @Override public Headline getHeadline() { - return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build()); + return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node) + .findFirst() + .orElse(Headline.builder().build()); } private Stream streamAllNodes() { - return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode); + return documentTree.allEntriesInOrder() + .map(DocumentTree.Entry::getNode); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java index e8e43d1f..14485df 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -26,6 +27,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Footer implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); + List treeId; TextBlock leafTextBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java index 2092c32..9285490 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Header implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; TextBlock leafTextBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java index 95be162..b708c2b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Headline implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; TextBlock leafTextBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java index 1a9c42d..bcfb039 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -29,6 +30,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Image implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); + List treeId; String id; @@ -66,7 +70,9 @@ public class Image implements GenericSemanticNode { @Override public TextBlock getTextBlock() { - return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + return streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java index e863c1a..24cb5c8 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java @@ -6,6 +6,8 @@ public enum ImageType { LOGO, FORMULA, SIGNATURE, + + SIGNATURE_VISUAL, OTHER, OCR; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java index 224b537..71010b6 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -24,6 +25,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Paragraph implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); + List treeId; TextBlock leafTextBlock; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java index 60cc243..6680c01 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -27,6 +28,8 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(level = AccessLevel.PRIVATE) public class Section implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; TextBlock textBlock; @@ -50,7 +53,8 @@ public class Section implements GenericSemanticNode { public boolean hasTables() { - return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent(); + return streamAllSubNodesOfType(NodeType.TABLE).findAny() + .isPresent(); } @@ -58,7 +62,9 @@ public class Section implements GenericSemanticNode { public TextBlock getTextBlock() { if (textBlock == null) { - textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } return textBlock; } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java index 811ca45..e35a83e 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java @@ -12,6 +12,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; @@ -335,6 +336,23 @@ public interface SemanticNode { } + /** + * returns the set of layoutengines. + * + * @return set of layoutengines. + */ + Set getEngines(); + + + /** + * adds a layoutengine to the set. + */ + default void addEngine(LayoutEngine engine) { + + getEngines().add(engine); + } + + /** * Streams all children located directly underneath this node in the DocumentTree. * @@ -426,6 +444,7 @@ public interface SemanticNode { /** * TODO: this produces unwanted results for sections spanning multiple columns. * Computes the Union of the bounding boxes of all children recursively. + * * @return The union of the BoundingBoxes of all children */ private Map getBBoxFromChildren() { diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java index 8f77162..5bed37b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java @@ -12,6 +12,7 @@ import java.util.Set; import java.util.stream.IntStream; import java.util.stream.Stream; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -31,6 +32,8 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Table implements SemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; DocumentTree documentTree; @@ -45,6 +48,7 @@ public class Table implements SemanticNode { @EqualsAndHashCode.Exclude Map bBoxCache; + /** * Streams all entities in this table, that appear in a row, which contains any of the provided strings. * @@ -53,8 +57,7 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowContainsStringsIgnoreCase(List strings) { - return IntStream.range(0, numberOfRows) - .boxed() + return IntStream.range(0, numberOfRows).boxed() .filter(row -> rowContainsStringsIgnoreCase(row, strings)) .flatMap(this::streamRow) .map(TableCell::getEntities) @@ -71,8 +74,11 @@ public class Table implements SemanticNode { */ public boolean rowContainsStringsIgnoreCase(Integer row, List strings) { - String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT); - return strings.stream().map(String::toLowerCase).allMatch(rowText::contains); + String rowText = streamRow(row).map(TableCell::getTextBlock) + .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT); + return strings.stream() + .map(String::toLowerCase) + .allMatch(rowText::contains); } @@ -85,9 +91,13 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowHasHeaderAndValue(String header, String value) { - List vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList(); + List vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)) + .map(TableCell::getCol) + .toList(); return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream() - .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream); + .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))) + .map(TableCell::getEntities) + .flatMap(Collection::stream); } @@ -100,9 +110,13 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List values) { - List colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList(); + List colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)) + .map(TableCell::getCol) + .toList(); return streamTableCells().filter(tableCellNode -> colsWithHeader.stream() - .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream); + .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))) + .map(TableCell::getEntities) + .flatMap(Collection::stream); } @@ -114,12 +128,15 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowContainsEntitiesOfType(List types) { - List rowsWithEntityOfType = IntStream.range(0, numberOfRows) - .boxed() - .filter(rowNumber -> streamEntityTypesInRow(rowNumber).anyMatch(existingType -> types.stream().anyMatch(typeToCheck -> typeToCheck.equals(existingType)))) + List rowsWithEntityOfType = IntStream.range(0, numberOfRows).boxed() + .filter(rowNumber -> streamEntityTypesInRow(rowNumber).anyMatch(existingType -> types.stream() + .anyMatch(typeToCheck -> typeToCheck.equals(existingType)))) .toList(); - return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream); + return rowsWithEntityOfType.stream() + .flatMap(this::streamRow) + .map(TableCell::getEntities) + .flatMap(Collection::stream); } @@ -131,18 +148,24 @@ public class Table implements SemanticNode { */ public Stream streamEntitiesWhereRowContainsNoEntitiesOfType(List types) { - List rowsWithNoEntityOfType = IntStream.range(0, numberOfRows) - .boxed() - .filter(rowNumber -> streamEntityTypesInRow(rowNumber).noneMatch(existingType -> types.stream().anyMatch(typeToCheck -> typeToCheck.equals(existingType)))) + List rowsWithNoEntityOfType = IntStream.range(0, numberOfRows).boxed() + .filter(rowNumber -> streamEntityTypesInRow(rowNumber).noneMatch(existingType -> types.stream() + .anyMatch(typeToCheck -> typeToCheck.equals(existingType)))) .toList(); - return rowsWithNoEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream); + return rowsWithNoEntityOfType.stream() + .flatMap(this::streamRow) + .map(TableCell::getEntities) + .flatMap(Collection::stream); } private Stream streamEntityTypesInRow(Integer rowNumber) { - return streamRow(rowNumber).map(TableCell::getEntities).flatMap(Collection::stream).map(RedactionEntity::getType).distinct(); + return streamRow(rowNumber).map(TableCell::getEntities) + .flatMap(Collection::stream) + .map(RedactionEntity::getType) + .distinct(); } @@ -159,7 +182,8 @@ public class Table implements SemanticNode { throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols)); } int idx = row * numberOfCols + col; - return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode(); + return (TableCell) documentTree.getEntryById(treeId).getChildren() + .get(idx).getNode(); } @@ -196,7 +220,8 @@ public class Table implements SemanticNode { */ public Stream streamCol(int col) { - return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col)); + return IntStream.range(0, numberOfRows).boxed() + .map(row -> getCell(row, col)); } @@ -208,9 +233,11 @@ public class Table implements SemanticNode { */ public Stream streamRow(int row) { - return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col)); + return IntStream.range(0, numberOfCols).boxed() + .map(col -> getCell(row, col)); } + /** * Streams all TableCells row-wise and filters them with header == true. * @@ -231,7 +258,8 @@ public class Table implements SemanticNode { */ public Stream streamHeadersForCell(int row, int col) { - return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader); + return Stream.concat(streamRow(row), streamCol(col)) + .filter(TableCell::isHeader); } @@ -304,7 +332,9 @@ public class Table implements SemanticNode { public TextBlock getTextBlock() { if (textBlock == null) { - textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } return textBlock; } @@ -315,6 +345,8 @@ public class Table implements SemanticNode { return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary(); } + + @Override public Map getBBox() { @@ -323,4 +355,5 @@ public class Table implements SemanticNode { } return bBoxCache; } + } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java index 3036d94..73ba2d5 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity; @@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class TableCell implements GenericSemanticNode { + @Builder.Default + Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); List treeId; int row; int col; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java index 56c36a2..c57fe9a 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java @@ -4,18 +4,21 @@ import java.awt.geom.Rectangle2D; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NonNull; import lombok.RequiredArgsConstructor; @Data @RequiredArgsConstructor +@AllArgsConstructor public class ClassifiedImage { @NonNull private Rectangle2D position; @NonNull private ImageType imageType; + private boolean sourceByAi; private boolean isAppendedToSection; @NonNull private boolean hasTransparency; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java index f91364d..6fe668b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java @@ -24,6 +24,7 @@ public class VisualLayoutParsingAdapter { private static String SIGNATURES = "signature"; + public Map> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) { Map> tableCells = new HashMap<>(); @@ -34,12 +35,17 @@ public class VisualLayoutParsingAdapter { } + public Map> buildExtractedSignaturesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) { Map> signatures = new HashMap<>(); - visualLayoutParsingResponse.getData().forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes()))); + if (visualLayoutParsingResponse.getData() != null) { + visualLayoutParsingResponse.getData() + .forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx() + 1, tableCell -> new ArrayList<>()) + .addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes()))); + } + return signatures; - return signatures; } @@ -63,14 +69,17 @@ public class VisualLayoutParsingAdapter { } + public List convertSignatures(int pageNumber, List tableObjects) { List signatures = new ArrayList<>(); tableObjects.stream().forEach(t -> { - if(t.getLabel().equals(SIGNATURES)) { - ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(),t.getBox().getY1(),t.getBox().getX2() - t.getBox().getX1(),t.getBox().getY2() - t.getBox().getY1()), - ImageType.SIGNATURE,false,pageNumber); + if (t.getLabel().equals(SIGNATURES)) { + ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(), + t.getBox().getY1(), + t.getBox().getX2() - t.getBox().getX1(), + t.getBox().getY2() - t.getBox().getY1()), ImageType.SIGNATURE, true, false, false, pageNumber); signatures.add(signature); } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java index 6bde310..778ad91 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java @@ -7,19 +7,19 @@ import static java.util.stream.Collectors.toList; import java.awt.geom.Rectangle2D; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage; -import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage; -import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock; import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer; @@ -31,6 +31,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section; import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock; +import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage; +import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock; import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder; import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations; @@ -95,14 +97,17 @@ public class DocumentGraphFactory { Rectangle2D position = image.getPosition(); Page page = context.getPage(image.getPage()); - Image imageNode = Image.builder() + var imageBuilder = Image.builder() .id(IdBuilder.buildId(Set.of(page), List.of(position))) .imageType(image.getImageType()) .position(position) .transparent(image.isHasTransparency()) .page(page) - .documentTree(context.getDocumentTree()) - .build(); + .documentTree(context.getDocumentTree()); + if (image.isSourceByAi()) { + imageBuilder.engines(new HashSet<>(Set.of(LayoutEngine.AI))); + } + Image imageNode = imageBuilder.build(); page.getMainBody().add(imageNode); List tocId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java index e58a176..4f05c1b 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/DocumentDataMapper.java @@ -80,6 +80,7 @@ public class DocumentDataMapper { .treeId(toPrimitiveIntArray(entry.getTreeId())) .children(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList()) .type(entry.getType()) + .engines(entry.getNode().getEngines()) .atomicBlockIds(atomicTextBlocks) .pageNumbers(entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new)) .properties(properties) diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java index a2ebcc0..36ffaa1 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java @@ -17,6 +17,7 @@ import java.util.stream.Stream; import org.springframework.stereotype.Service; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page; @@ -58,48 +59,59 @@ public class LayoutGridService { static Color HEADER_COLOR = new Color(171, 131, 6); static Color IMAGE_COLOR = new Color(253, 63, 146); + static Color IMAGE_VISUAL_COLOR = new Color(122, 0, 255); + @SneakyThrows @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document") public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) { - LayoutGrid layoutGrid = createLayoutGrid(document); + this.addLayoutGrid(originFile, document, destinationFile, layerVisibilityDefaultValue, false); + } + + + @SneakyThrows + @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document") + public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue, boolean visualParsingGrid) { + + LayoutGrid layoutGrid = createLayoutGrid(document, visualParsingGrid); viewerDocumentService.addVisualizationsOnPage(originFile, destinationFile, Visualizations.builder() - .layer(ContentStreams.KNECON_LAYOUT) + .layer(visualParsingGrid ? ContentStreams.KNECON_VISUAL_PARSING : ContentStreams.KNECON_LAYOUT) .visualizationsOnPages(layoutGrid.getVisualizationsPerPages()) .layerVisibilityDefaultValue(layerVisibilityDefaultValue) .build()); } - private LayoutGrid createLayoutGrid(Document document) { + private LayoutGrid createLayoutGrid(Document document, boolean visualParsingGrid) { LayoutGrid layoutGrid = new LayoutGrid(document.getNumberOfPages()); - document.streamAllSubNodes().forEach(semanticNode -> { - Color color = switch (semanticNode.getType()) { - case PARAGRAPH -> PARAGRAPH_COLOR; - case TABLE -> TABLE_COLOR; - case SECTION -> SECTION_COLOR; - case HEADLINE -> HEADLINE_COLOR; - case HEADER, FOOTER -> HEADER_COLOR; - case IMAGE -> IMAGE_COLOR; - default -> null; - }; - if (isNotSectionOrTableCellOrDocument(semanticNode)) { - addAsRectangle(semanticNode, layoutGrid, color); - } - if (semanticNode.getType().equals(NodeType.SECTION)) { - addSection(semanticNode, layoutGrid, color); - } - if (semanticNode.getType().equals(NodeType.TABLE)) { - Table table = (Table) semanticNode; - addInnerTableLines(table, layoutGrid); - } - }); - + document.streamAllSubNodes() + .filter(node -> (node.getEngines().contains(LayoutEngine.AI) && visualParsingGrid) || (node.getEngines().contains(LayoutEngine.ALGORITHM) && !visualParsingGrid)) + .forEach(semanticNode -> { + Color color = switch (semanticNode.getType()) { + case PARAGRAPH -> PARAGRAPH_COLOR; + case TABLE -> TABLE_COLOR; + case SECTION -> SECTION_COLOR; + case HEADLINE -> HEADLINE_COLOR; + case HEADER, FOOTER -> HEADER_COLOR; + case IMAGE -> IMAGE_COLOR; + default -> null; + }; + if (isNotSectionOrTableCellOrDocument(semanticNode)) { + addAsRectangle(semanticNode, layoutGrid, color); + } + if (semanticNode.getType().equals(NodeType.SECTION)) { + addSection(semanticNode, layoutGrid, color); + } + if (semanticNode.getType().equals(NodeType.TABLE)) { + Table table = (Table) semanticNode; + addInnerTableLines(table, layoutGrid); + } + }); return layoutGrid; } diff --git a/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java b/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java index 882e72b..937f75d 100644 --- a/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java +++ b/layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/ContentStreams.java @@ -12,6 +12,8 @@ public class ContentStreams { public static Identifier KNECON_LAYOUT = new Identifier("Layout grid", COSName.getPDFName("KNECON_LAYOUT"), true); + public static Identifier KNECON_VISUAL_PARSING = new Identifier("Layout grid - visual", COSName.getPDFName("KNECON_VISUAL_PARSING"), true); + public static Identifier KNECON_OCR = new Identifier("OCR", COSName.getPDFName("KNECON_OCR"), false); public static Identifier KNECON_OCR_TEXT_DEBUG = new Identifier("OCR Text", COSName.getPDFName("KNECON_OCR_TEXT_DEBUG"), true); @@ -24,7 +26,14 @@ public class ContentStreams { public static Identifier ESCAPE_END = new Identifier("escape start", COSName.getPDFName("ESCAPE_END"), false); - public static List allContentStreams = List.of(KNECON_LAYOUT, KNECON_OCR, KNECON_OCR_BBOX_DEBUG, KNECON_OCR_TEXT_DEBUG, OTHER, ESCAPE_START, ESCAPE_END); + public static List allContentStreams = List.of(KNECON_LAYOUT, + KNECON_VISUAL_PARSING, + KNECON_OCR, + KNECON_OCR_BBOX_DEBUG, + KNECON_OCR_TEXT_DEBUG, + OTHER, + ESCAPE_START, + ESCAPE_END); public record Identifier(String name, COSName cosName, boolean optionalContent) {