Marmelator/layout-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 289 Wiki Activity

RED-6009: Document Tree Structure

Browse Source 
*refactored mappers to be static
*added storage auto configuration
This commit is contained in:
Kilian Schuettler 2023-04-12 14:10:39 +02:00
parent 2ed617bb03
commit e13b5af449
5 changed files with 15 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
layoutparser-service/layoutparser-service-internal-api/src/main/java/com.knecon.fforesight.service.layoutparser.internal.api/mapper/DocumentDataMapper.java
View File

@ -19,6 +19,9 @@ import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.Table
import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
import lombok.experimental.UtilityClass;
@UtilityClass
public class DocumentDataMapper {
public DocumentData toDocumentData(DocumentGraph documentGraph) {
@ -26,16 +29,16 @@ public class DocumentDataMapper {
List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder()
.flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
.distinct()
.map(this::toAtomicTextBlockData)
.map(DocumentDataMapper::toAtomicTextBlockData)
.toList();
List<AtomicPositionBlockData> atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder()
.flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
.distinct()
.map(this::toAtomicPositionBlockData)
.map(DocumentDataMapper::toAtomicPositionBlockData)
.toList();
List<PageData> pageData = documentGraph.getPages().stream().map(this::toPageData).toList();
List<PageData> pageData = documentGraph.getPages().stream().map(DocumentDataMapper::toPageData).toList();
TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
return DocumentData.builder()
.atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
@ -71,7 +74,7 @@ public class DocumentDataMapper {
return TableOfContentsData.EntryData.builder()
.tocId(toPrimitiveIntArray(entry.tocId()))
.subEntries(entry.children().stream().map(this::toEntryData).toList())
.subEntries(entry.children().stream().map(DocumentDataMapper::toEntryData).toList())
.type(entry.type())
.atomicBlocks(atomicTextBlocks)
.pages(entry.node().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new))

5
layoutparser-service/layoutparser-service-internal-api/src/main/java/com.knecon.fforesight.service.layoutparser.internal.api/mapper/DocumentGraphMapper.java
View File

@ -34,6 +34,9 @@ import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.A
import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
import lombok.experimental.UtilityClass;
@UtilityClass
public class DocumentGraphMapper {
public DocumentGraph toDocumentGraph(DocumentData documentData) {
@ -47,7 +50,7 @@ public class DocumentGraphMapper {
Arrays.stream(documentData.getAtomicTextBlocks()).toList(),
Arrays.stream(documentData.getAtomicPositionBlocks()).toList());
context.pages.addAll(Arrays.stream(documentData.getPages()).map(this::buildPage).toList());
context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList());
context.tableOfContents.getRoot().children().addAll(buildEntries(documentData.getTableOfContents().getRoot().subEntries(), context));

3
layoutparser-service/layoutparser-service-server/src/main/java/com.knecon.fforesight.service.layoutparser.server/Application.java
View File

@ -8,12 +8,13 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Import;
import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.LayoutparserServiceProcessorConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.AsyncConfig;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.MultiTenancyMessagingConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.multitenancy.MultiTenancyWebConfiguration;
@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class, MetricsConfiguration.class, LayoutparserServiceProcessorConfiguration.class})
@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class, MetricsConfiguration.class, LayoutparserServiceProcessorConfiguration.class, StorageAutoConfiguration.class})
@EnableFeignClients
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
public class Application {

13
layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphMappingTest.java
View File

@ -1,7 +1,6 @@
package com.knecon.fforesight.service.layoutparser.server.graph;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
@ -13,14 +12,6 @@ import lombok.SneakyThrows;
public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
@Autowired
private DocumentDataMapper documentDataMapper;
@Autowired
private DocumentGraphMapper documentGraphMapper;
@Test
@SneakyThrows
public void testGraphMapping() {
@ -28,10 +19,10 @@ public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
DocumentGraph document = buildGraph("files/crafted document");
LayoutParsingRequest layoutParsingRequest = buildStandardLayoutParsingRequest();
DocumentData documentData = documentDataMapper.toDocumentData(document);
DocumentData documentData = DocumentDataMapper.toDocumentData(document);
layoutParsingStorageService.storeDocumentData(layoutParsingRequest, documentData);
DocumentData documentData2 = layoutParsingStorageService.readDocumentData(layoutParsingRequest);
DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData2);
DocumentGraph newDocumentGraph = DocumentGraphMapper.toDocumentGraph(documentData2);
assert document.toString().equals(newDocumentGraph.toString());
assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString());

16
layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BaseTest.java
View File

@ -21,8 +21,6 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.mapper.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.mapper.DocumentGraphMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.services.EntityEnrichmentService;
import com.knecon.fforesight.service.layoutparser.internal.api.services.EntityInsertionService;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
@ -163,20 +161,6 @@ public class BaseTest {
}
@Bean
public DocumentDataMapper documentDataMapper() {
return new DocumentDataMapper();
}
@Bean
public DocumentGraphMapper documentGraphMapper() {
return new DocumentGraphMapper();
}
@Bean
@Autowired
public EntityInsertionService entityInsertionService(EntityEnrichmentService entityEnrichmentService) {