Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009: Document Tree Structure

Browse Source 
* mapped pages back into ImageNodes in the graphMapper
* fixed path in the noExceptionShouldBeThrownForAnyFilesTest
This commit is contained in:
Kilian Schuettler 2023-04-27 11:56:00 +02:00
parent c7b08410e1
commit 2f9b023bb4
3 changed files with 30 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java
View File

@ -78,7 +78,7 @@ public class DocumentGraphMapper {
case FOOTER -> buildFooter(context, terminal); case FOOTER -> buildFooter(context, terminal);
case TABLE -> buildTable(context, entryData.getProperties()); case TABLE -> buildTable(context, entryData.getProperties());
case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal); case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal);
case IMAGE -> buildImage(context, entryData.getProperties()); case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPages());
default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType()); default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
}; };
@ -114,11 +114,13 @@ public class DocumentGraphMapper {
} }
private ImageNode buildImage(Context context, Map<String, String> properties) { private ImageNode buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) {
assert pageNumbers.length == 1;
PageNode pageNode = getPage(pageNumbers[0], context);
var builder = ImageNode.builder(); var builder = ImageNode.builder();
PropertiesMapper.parseImageProperties(properties, builder); PropertiesMapper.parseImageProperties(properties, builder);
return builder.tableOfContents(context.tableOfContents()).build(); return builder.tableOfContents(context.tableOfContents()).page(pageNode).build();
} }

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java
View File

@ -53,9 +53,7 @@ public class SectionNodeFactory {
} }
} }
for (ClassifiedImage image : images) { images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(sectionNode, image, context));
DocumentGraphFactory.addImage(sectionNode, image, context);
}
} }

48
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -206,8 +206,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test @Test
public void titleExtraction() throws IOException { public void titleExtraction() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
System.out.println("Start Full integration test"); System.out.println("Start Full integration test");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
System.out.println("Finished structure analysis"); System.out.println("Finished structure analysis");
@ -287,36 +286,37 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
URL url = loader.getResource("files"); URL url = loader.getResource("files");
Path path = Paths.get(URI.create(url.toString())); Path path = Paths.get(URI.create(url.toString()));
Files.walk(path).forEach(currentPath -> { Files.walk(path)//
AnalyzeRequest request = uploadFileToStorage(currentPath.toString()); .filter(currentPath -> currentPath.toString().endsWith(".pdf"))//
System.out.println("Redacting file : " + currentPath.getFileName()); .map(currentPath -> path.getParent().relativize(currentPath))//
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); .forEach(currentPath -> {
AnalyzeRequest request = uploadFileToStorage(currentPath.toString());
System.out.println("Redacting file : " + currentPath.getFileName());
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
long fstart = System.currentTimeMillis(); long fstart = System.currentTimeMillis();
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
System.out.println("analysis analysis duration: " + (System.currentTimeMillis() - fstart)); System.out.println("analysis analysis duration: " + (System.currentTimeMillis() - fstart));
Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>(); Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getRedactionLogEntry().forEach(entry -> { redactionLog.getRedactionLogEntry().forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
}); });
duplicates.entrySet().forEach(entry -> { duplicates.forEach((id, redactionLogEntries) -> assertThat(redactionLogEntries.size()).isEqualTo(1));
assertThat(entry.getValue().size()).isEqualTo(1);
});
dictionary.get(DICTIONARY_AUTHOR).add("Drinking water"); dictionary.get(DICTIONARY_AUTHOR).add("Drinking water");
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
long rstart = System.currentTimeMillis(); long rstart = System.currentTimeMillis();
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
long rend = System.currentTimeMillis(); long rend = System.currentTimeMillis();
System.out.println("reanalysis analysis duration: " + (rend - rstart)); System.out.println("reanalysis analysis duration: " + (rend - rstart));
}); });
long end = System.currentTimeMillis(); long end = System.currentTimeMillis();