diff --git a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java index 6d4c4c6..c364bbc 100644 --- a/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java +++ b/layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java @@ -36,9 +36,6 @@ public record LayoutParsingRequest( @Schema(description = "Path where the Simplified Text File will be stored.")// @NonNull String simplifiedTextStorageId,// @Schema(description = "Path where the Viewer Document PDF will be stored.")// - @NonNull String viewerDocumentStorageId,// - @Deprecated// - @Schema(description = "Path where the Section Grid will be stored.")// - @NonNull String sectionGridStorageId) { + @NonNull String viewerDocumentStorageId) { } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java index b14fb8a..8418e89 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java @@ -35,7 +35,6 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse; import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService; import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService; -import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService; import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService; import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService; import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService; @@ -65,7 +64,6 @@ public class LayoutParsingPipeline { private final CvTableParsingAdapter cvTableParsingAdapter; private final LayoutParsingStorageService layoutParsingStorageService; private final SectionsBuilderService sectionsBuilderService; - private final SectionGridCreatorService sectionGridCreatorService; private final TaasClassificationService taasClassificationService; private final RedactManagerClassificationService redactManagerClassificationService; private final DocuMineClassificationService docuMineClassificationService; @@ -99,7 +97,6 @@ public class LayoutParsingPipeline { int numberOfPages = originDocument.getNumberOfPages(); - layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph)); layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph)); layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph)); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java index 9575d04..6e35cbe 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java @@ -17,7 +17,6 @@ import org.apache.pdfbox.pdmodel.PDDocument; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText; @@ -84,11 +83,6 @@ public class LayoutParsingStorageService { } - public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) { - - storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid); - } - public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) { diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationDocument.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationDocument.java index 5062790..b3565ae 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationDocument.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationDocument.java @@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.model; import java.util.ArrayList; import java.util.List; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter; import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText; @@ -25,7 +24,6 @@ public class ClassificationDocument { private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter(); private boolean headlines; - private SectionGrid sectionGrid = new SectionGrid(); private long rulesVersion; } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionGridCreatorService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionGridCreatorService.java deleted file mode 100644 index f159583..0000000 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionGridCreatorService.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.knecon.fforesight.service.layoutparser.processor.services; - -import java.awt.geom.Rectangle2D; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.BinaryOperator; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collector; -import java.util.stream.Stream; - -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; -import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document; -import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page; -import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode; -import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table; -import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell; - -import lombok.RequiredArgsConstructor; - -@Service -@RequiredArgsConstructor -public class SectionGridCreatorService { - - public SectionGrid createSectionGrid(Document document) { - - Map> sectionBBox = document.streamAllSubNodesOfType(NodeType.SECTION).map(SemanticNode::getBBox).collect(new SectionGridCollector()); - Map> paragraphBBox = document.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBBox).collect(new SectionGridCollector()); - Map> headlineBBox = document.streamAllSubNodesOfType(NodeType.HEADLINE).map(SemanticNode::getBBox).collect(new SectionGridCollector()); - Map> tableBBox = document.streamAllSubNodesOfType(NodeType.TABLE).map(node -> (Table) node).collect(new TableGridCollector()); - var sectionGrid = new SectionGrid(); - - sectionGrid.setRectanglesPerPage(mergeMapsByConcatenatingLists(// - mergeMapsByConcatenatingLists(paragraphBBox, headlineBBox), // - mergeMapsByConcatenatingLists(sectionBBox, tableBBox))); - - return sectionGrid; - } - - - private static abstract class GridCollector implements Collector>, Map>> { - - @Override - public Supplier>> supplier() { - - return HashMap::new; - } - - - @Override - public Function>, Map>> finisher() { - - return Function.identity(); - } - - - @Override - public BinaryOperator>> combiner() { - - return SectionGridCreatorService::mergeMapsByConcatenatingLists; - } - - - @Override - public Set characteristics() { - - return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED); - } - - } - - private static class TableGridCollector extends GridCollector { - - @Override - public BiConsumer>, Table> accumulator() { - - return (map, table) -> table.getPages() - .forEach(page -> map.merge(page.getNumber(), List.of(toSectionRectangle(table, page, table.getPages().size())), SectionGridCreatorService::concatLists)); - } - - - private static SectionRectangle toSectionRectangle(Table table, Page page, int numberOfParts) { - - Rectangle2D rect = table.getBBox().get(page); - List tableCellRectangles = table.streamTableCells() - .map(TableCell::getBBox) - .map(map -> map.get(page)) - .filter(Objects::nonNull) - .map(rectangle2D -> new CellRectangle(new Point((float) rectangle2D.getX(), (float) rectangle2D.getY()), - (float) rectangle2D.getWidth(), - (float) rectangle2D.getHeight())) - .toList(); - return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()), - (float) rect.getWidth(), - (float) rect.getHeight(), - 1, - numberOfParts, - tableCellRectangles); - } - - } - - private static class SectionGridCollector extends GridCollector> { - - @Override - public BiConsumer>, Map> accumulator() { - - return (mapToKeep, mapToMerge) -> mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page.getNumber(), - List.of(toSectionRectangle(rectangle, mapToMerge.values().size())), - SectionGridCreatorService::concatLists)); - - } - - - private static SectionRectangle toSectionRectangle(Rectangle2D rect, int numberOfParts) { - - return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()), (float) rect.getWidth(), (float) rect.getHeight(), 1, numberOfParts, null); - } - - } - - - private static Map> mergeMapsByConcatenatingLists(Map> mapToKeep, - Map> mapToMerge) { - - mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page, rectangle, SectionGridCreatorService::concatLists)); - return mapToKeep; - } - - - private static List concatLists(List l1, List l2) { - - return Stream.concat(l1.stream(), l2.stream()).toList(); - } - -} diff --git a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java index 3740351..4f56c76 100644 --- a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java +++ b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java @@ -51,7 +51,6 @@ public abstract class AbstractTest { protected final static String POSITION_FILE_ID = "positions"; protected final static String PAGES_FILE_ID = "pages"; protected final static String TENANT_ID = "tenant"; - protected final static String SECTION_GRID_ID = "section"; protected final static String VIEWER_DOCUMENT_ID = "viewer"; protected final static String SIMPLIFIED_ID = "simplified"; @@ -68,7 +67,6 @@ public abstract class AbstractTest { .positionBlockFileStorageId(POSITION_FILE_ID) .pageFileStorageId(PAGES_FILE_ID) .simplifiedTextStorageId(SIMPLIFIED_ID) - .sectionGridStorageId(SECTION_GRID_ID) .viewerDocumentStorageId(VIEWER_DOCUMENT_ID) .build(); } @@ -114,7 +112,6 @@ public abstract class AbstractTest { .positionBlockFileStorageId(POSITION_FILE_ID) .pageFileStorageId(PAGES_FILE_ID) .simplifiedTextStorageId(SIMPLIFIED_ID) - .sectionGridStorageId(SECTION_GRID_ID) .viewerDocumentStorageId(VIEWER_DOCUMENT_ID) .build(); }