Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

hotfix: Document parsing

Browse Source
This commit is contained in:
deiflaender 2023-04-17 13:08:54 +02:00
parent 74362f4233
commit e4a034700a
3 changed files with 105 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java
View File

@ -12,6 +12,7 @@ import lombok.NoArgsConstructor;
public class RedactionResult {
private byte[] document;
private byte[] jsonDoc;
private int numberOfPages;
}

56
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.controller;
import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.stream.Collectors;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.web.bind.annotation.PathVariable;
@ -72,29 +73,55 @@ public class RedactionController implements RedactionResource {
var t = page.getTextBlocks().get(i);
if(t instanceof TextBlock){
var textBlock = (TextBlock) t;
document.getParagraphs()
.add(Paragraph.builder()
.text(t.getText())
.boundingBox(new Rectangle(new Point(t.getMinX(), t.getMinY()), t.getWidth(), t.getHeight(), t.getPage()))
.textStyle(((TextBlock) t).getMostPopularWordStyle())
.text(textBlock.getText())
.boundingBox(new Rectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock.getHeight(), textBlock.getPage()))
.textStyle(textBlock.getMostPopularWordStyle())
.paragraphNum(i)
.orientation(t.getOrientation())
.classification(t.getClassification())
.textDirection(((TextBlock) t).getSequences().get(0).getDir())
.page(t.getPage())
.orientation(textBlock.getOrientation())
.classification(textBlock.getClassification())
.textDirection(textBlock.getSequences().get(0).getDir())
.page(textBlock.getPage())
.build());
} else {
var table = (Table) t;
for (var row: table.getRows()){
StringBuilder sb = new StringBuilder();
row.forEach(cell -> {
sb.append(cell.getTextBlocks().stream().map(a -> a.getText()).collect(Collectors.joining("|")));
});
document.getParagraphs()
.add(Paragraph.builder()
.text(sb.toString())
.boundingBox(new Rectangle(new Point(table.getMinX(), table.getMinY()), table.getWidth(), table.getHeight(), table.getPage()))
.paragraphNum(i)
.orientation(table.getOrientation())
.classification("Table Row")
.page(table.getPage())
.build());
}
}
}
});
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Document.json")) {
fileOutputStream.write(objectMapper.writeValueAsBytes(document));
}
pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
return convert(pdDocument, classifiedDoc.getPages().size());
return convert(pdDocument, classifiedDoc.getPages().size(), objectMapper.writeValueAsBytes(document));
} catch (IOException e) {
throw new RedactionException(e);
@ -123,7 +150,7 @@ public class RedactionController implements RedactionResource {
pdDocument.setAllSecurityToBeRemoved(true);
pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
return convert(pdDocument, classifiedDoc.getPages().size());
return convert(pdDocument, classifiedDoc.getPages().size(), null);
} catch (IOException e) {
throw new RedactionException(e);
@ -183,11 +210,12 @@ public class RedactionController implements RedactionResource {
}
private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
private RedactionResult convert(PDDocument document, int numberOfPages, byte[] json) throws IOException {
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
document.save(byteArrayOutputStream);
return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages)
.jsonDoc(json).build();
}
}

55
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -6,12 +6,17 @@ import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
@ -24,6 +29,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@ -1111,7 +1117,13 @@ public class RedactionIntegrationTest {
System.out.println("classificationTest");
AnalyzeRequest request = prepareStorage("files/new/2001160.pdf");
String sourcePath ="/tmp";
String targetPath = "/tmp/result";
var files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(sourcePath), sourcePath);
for(var file : files){
AnalyzeRequest request = prepareStorage(file);
RedactionRequest redactionRequest = RedactionRequest.builder()
.dossierId(request.getDossierId())
@ -1121,10 +1133,44 @@ public class RedactionIntegrationTest {
RedactionResult result = redactionController.classify(redactionRequest);
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
String filename = file.substring(file.lastIndexOf("/"));
try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename)) {
fileOutputStream.write(result.getDocument());
}
try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename + ".json")) {
fileOutputStream.write(result.getJsonDoc());
}
}
}
@SneakyThrows
private Set<String> getFileNames(Set<String> fileNames, Path dir, String resourcePath) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
for (Path path : stream) {
if (path.toFile().isDirectory()) {
getFileNames(fileNames, path, resourcePath);
} else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
String absolutePath = cleanPath(path.toAbsolutePath().toString());
fileNames.add(absolutePath);
}
}
}
return fileNames;
}
private String cleanPath(String path) {
return StringUtils.replace(path, "\\", "/");
}
@Test
@ -1793,10 +1839,11 @@ public class RedactionIntegrationTest {
@SneakyThrows
private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
ClassPathResource pdfFileResource = new ClassPathResource(file);
File initialFile = new File(file);
InputStream targetStream = new FileInputStream(initialFile);
ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
return prepareStorage(targetStream, cvServiceResponseFileResource.getInputStream());
}
}