Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009: Document Tree Structure

Browse Source 
* mapped pages back into ImageNodes in the graphMapper
* fixed path in the noExceptionShouldBeThrownForAnyFilesTest
This commit is contained in:
Kilian Schuettler 2023-04-27 11:56:00 +02:00
parent 7ed5ba759e
commit 8e9700fca0
3 changed files with 30 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java
View File

@ -78,7 +78,7 @@ public class DocumentGraphMapper {
case FOOTER -> buildFooter(context, terminal);
case TABLE -> buildTable(context, entryData.getProperties());
case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal);
case IMAGE -> buildImage(context, entryData.getProperties());
case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPages());
default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
};
@ -114,11 +114,13 @@ public class DocumentGraphMapper {
}
private ImageNode buildImage(Context context, Map<String, String> properties) {
private ImageNode buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) {
assert pageNumbers.length == 1;
PageNode pageNode = getPage(pageNumbers[0], context);
var builder = ImageNode.builder();
PropertiesMapper.parseImageProperties(properties, builder);
return builder.tableOfContents(context.tableOfContents()).build();
return builder.tableOfContents(context.tableOfContents()).page(pageNode).build();
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java
View File

@ -53,9 +53,7 @@ public class SectionNodeFactory {
}
}
for (ClassifiedImage image : images) {
DocumentGraphFactory.addImage(sectionNode, image, context);
}
images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(sectionNode, image, context));
}

48
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -206,8 +206,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test
public void titleExtraction() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
System.out.println("Start Full integration test");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
System.out.println("Finished structure analysis");
@ -287,36 +286,37 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
URL url = loader.getResource("files");
Path path = Paths.get(URI.create(url.toString()));
Files.walk(path).forEach(currentPath -> {
AnalyzeRequest request = uploadFileToStorage(currentPath.toString());
System.out.println("Redacting file : " + currentPath.getFileName());
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
Files.walk(path)//
.filter(currentPath -> currentPath.toString().endsWith(".pdf"))//
.map(currentPath -> path.getParent().relativize(currentPath))//
.forEach(currentPath -> {
AnalyzeRequest request = uploadFileToStorage(currentPath.toString());
System.out.println("Redacting file : " + currentPath.getFileName());
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
long fstart = System.currentTimeMillis();
AnalyzeResult result = analyzeService.analyze(request);
System.out.println("analysis analysis duration: " + (System.currentTimeMillis() - fstart));
long fstart = System.currentTimeMillis();
AnalyzeResult result = analyzeService.analyze(request);
System.out.println("analysis analysis duration: " + (System.currentTimeMillis() - fstart));
Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getRedactionLogEntry().forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
});
redactionLog.getRedactionLogEntry().forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
});
duplicates.entrySet().forEach(entry -> {
assertThat(entry.getValue().size()).isEqualTo(1);
});
duplicates.forEach((id, redactionLogEntries) -> assertThat(redactionLogEntries.size()).isEqualTo(1));
dictionary.get(DICTIONARY_AUTHOR).add("Drinking water");
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
dictionary.get(DICTIONARY_AUTHOR).add("Drinking water");
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
long rstart = System.currentTimeMillis();
analyzeService.reanalyze(request);
long rstart = System.currentTimeMillis();
analyzeService.reanalyze(request);
long rend = System.currentTimeMillis();
System.out.println("reanalysis analysis duration: " + (rend - rstart));
});
long rend = System.currentTimeMillis();
System.out.println("reanalysis analysis duration: " + (rend - rstart));
});
long end = System.currentTimeMillis();