From 2f9b023bb46094ad5f3c9b6cc3b468ea8cad6e58 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 27 Apr 2023 11:56:00 +0200 Subject: [PATCH] RED-6009: Document Tree Structure * mapped pages back into ImageNodes in the graphMapper * fixed path in the noExceptionShouldBeThrownForAnyFilesTest --- .../data/mapper/DocumentGraphMapper.java | 8 ++-- .../document/factory/SectionNodeFactory.java | 4 +- .../v1/server/RedactionIntegrationTest.java | 48 +++++++++---------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java index 5e3b64c3..1d374053 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java @@ -78,7 +78,7 @@ public class DocumentGraphMapper { case FOOTER -> buildFooter(context, terminal); case TABLE -> buildTable(context, entryData.getProperties()); case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal); - case IMAGE -> buildImage(context, entryData.getProperties()); + case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPages()); default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType()); }; @@ -114,11 +114,13 @@ public class DocumentGraphMapper { } - private ImageNode buildImage(Context context, Map properties) { + private ImageNode buildImage(Context context, Map properties, Long[] pageNumbers) { + assert pageNumbers.length == 1; + PageNode pageNode = getPage(pageNumbers[0], context); var builder = ImageNode.builder(); PropertiesMapper.parseImageProperties(properties, builder); - return builder.tableOfContents(context.tableOfContents()).build(); + return builder.tableOfContents(context.tableOfContents()).page(pageNode).build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java index ee901820..e69c1cfd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java @@ -53,9 +53,7 @@ public class SectionNodeFactory { } } - for (ClassifiedImage image : images) { - DocumentGraphFactory.addImage(sectionNode, image, context); - } + images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(sectionNode, image, context)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index ed751c82..1041b33a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -206,8 +206,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Test public void titleExtraction() throws IOException { - AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); - + AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); System.out.println("Start Full integration test"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); System.out.println("Finished structure analysis"); @@ -287,36 +286,37 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { URL url = loader.getResource("files"); Path path = Paths.get(URI.create(url.toString())); - Files.walk(path).forEach(currentPath -> { - AnalyzeRequest request = uploadFileToStorage(currentPath.toString()); - System.out.println("Redacting file : " + currentPath.getFileName()); - analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + Files.walk(path)// + .filter(currentPath -> currentPath.toString().endsWith(".pdf"))// + .map(currentPath -> path.getParent().relativize(currentPath))// + .forEach(currentPath -> { + AnalyzeRequest request = uploadFileToStorage(currentPath.toString()); + System.out.println("Redacting file : " + currentPath.getFileName()); + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); - long fstart = System.currentTimeMillis(); - AnalyzeResult result = analyzeService.analyze(request); - System.out.println("analysis analysis duration: " + (System.currentTimeMillis() - fstart)); + long fstart = System.currentTimeMillis(); + AnalyzeResult result = analyzeService.analyze(request); + System.out.println("analysis analysis duration: " + (System.currentTimeMillis() - fstart)); - Map> duplicates = new HashMap<>(); + Map> duplicates = new HashMap<>(); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - redactionLog.getRedactionLogEntry().forEach(entry -> { - duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); - }); + redactionLog.getRedactionLogEntry().forEach(entry -> { + duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); + }); - duplicates.entrySet().forEach(entry -> { - assertThat(entry.getValue().size()).isEqualTo(1); - }); + duplicates.forEach((id, redactionLogEntries) -> assertThat(redactionLogEntries.size()).isEqualTo(1)); - dictionary.get(DICTIONARY_AUTHOR).add("Drinking water"); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L); + dictionary.get(DICTIONARY_AUTHOR).add("Drinking water"); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L); - long rstart = System.currentTimeMillis(); - analyzeService.reanalyze(request); + long rstart = System.currentTimeMillis(); + analyzeService.reanalyze(request); - long rend = System.currentTimeMillis(); - System.out.println("reanalysis analysis duration: " + (rend - rstart)); - }); + long rend = System.currentTimeMillis(); + System.out.println("reanalysis analysis duration: " + (rend - rstart)); + }); long end = System.currentTimeMillis();