From e4a034700ae0f2b1ba19476d47f06bda47925fd9 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Mon, 17 Apr 2023 13:08:54 +0200 Subject: [PATCH] hotfix: Document parsing --- .../redaction/v1/model/RedactionResult.java | 1 + .../controller/RedactionController.java | 64 ++++++++++++----- .../v1/server/RedactionIntegrationTest.java | 69 ++++++++++++++++--- 3 files changed, 105 insertions(+), 29 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java index 80650eab..3a657449 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java @@ -12,6 +12,7 @@ import lombok.NoArgsConstructor; public class RedactionResult { private byte[] document; + private byte[] jsonDoc; private int numberOfPages; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index b5022156..0aa07bd9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.controller; import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.util.stream.Collectors; import org.apache.pdfbox.pdmodel.PDDocument; import org.springframework.web.bind.annotation.PathVariable; @@ -72,29 +73,55 @@ public class RedactionController implements RedactionResource { var t = page.getTextBlocks().get(i); - document.getParagraphs() - .add(Paragraph.builder() - .text(t.getText()) - .boundingBox(new Rectangle(new Point(t.getMinX(), t.getMinY()), t.getWidth(), t.getHeight(), t.getPage())) - .textStyle(((TextBlock) t).getMostPopularWordStyle()) - .paragraphNum(i) - .orientation(t.getOrientation()) - .classification(t.getClassification()) - .textDirection(((TextBlock) t).getSequences().get(0).getDir()) - .page(t.getPage()) - .build()); + if(t instanceof TextBlock){ + + var textBlock = (TextBlock) t; + + document.getParagraphs() + .add(Paragraph.builder() + .text(textBlock.getText()) + .boundingBox(new Rectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock.getHeight(), textBlock.getPage())) + .textStyle(textBlock.getMostPopularWordStyle()) + .paragraphNum(i) + .orientation(textBlock.getOrientation()) + .classification(textBlock.getClassification()) + .textDirection(textBlock.getSequences().get(0).getDir()) + .page(textBlock.getPage()) + .build()); + } else { + + var table = (Table) t; + + for (var row: table.getRows()){ + + StringBuilder sb = new StringBuilder(); + + row.forEach(cell -> { + sb.append(cell.getTextBlocks().stream().map(a -> a.getText()).collect(Collectors.joining("|"))); + }); + + document.getParagraphs() + .add(Paragraph.builder() + .text(sb.toString()) + .boundingBox(new Rectangle(new Point(table.getMinX(), table.getMinY()), table.getWidth(), table.getHeight(), table.getPage())) + .paragraphNum(i) + .orientation(table.getOrientation()) + .classification("Table Row") + .page(table.getPage()) + .build()); + } + } + + } }); - try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Document.json")) { - fileOutputStream.write(objectMapper.writeValueAsBytes(document)); - } pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument); - return convert(pdDocument, classifiedDoc.getPages().size()); + return convert(pdDocument, classifiedDoc.getPages().size(), objectMapper.writeValueAsBytes(document)); } catch (IOException e) { throw new RedactionException(e); @@ -123,7 +150,7 @@ public class RedactionController implements RedactionResource { pdDocument.setAllSecurityToBeRemoved(true); pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument); - return convert(pdDocument, classifiedDoc.getPages().size()); + return convert(pdDocument, classifiedDoc.getPages().size(), null); } catch (IOException e) { throw new RedactionException(e); @@ -183,11 +210,12 @@ public class RedactionController implements RedactionResource { } - private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException { + private RedactionResult convert(PDDocument document, int numberOfPages, byte[] json) throws IOException { try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { document.save(byteArrayOutputStream); - return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build(); + return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages) + .jsonDoc(json).build(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 9a0399eb..f968a634 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -6,12 +6,17 @@ import static org.mockito.Mockito.when; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.ArrayList; @@ -24,6 +29,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -1111,22 +1117,62 @@ public class RedactionIntegrationTest { System.out.println("classificationTest"); - AnalyzeRequest request = prepareStorage("files/new/2001160.pdf"); + String sourcePath ="/tmp"; + String targetPath = "/tmp/result"; - RedactionRequest redactionRequest = RedactionRequest.builder() - .dossierId(request.getDossierId()) - .fileId(request.getFileId()) - .dossierTemplateId(request.getDossierTemplateId()) - .build(); + var files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(sourcePath), sourcePath); - RedactionResult result = redactionController.classify(redactionRequest); + for(var file : files){ + AnalyzeRequest request = prepareStorage(file); - try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) { - fileOutputStream.write(result.getDocument()); + RedactionRequest redactionRequest = RedactionRequest.builder() + .dossierId(request.getDossierId()) + .fileId(request.getFileId()) + .dossierTemplateId(request.getDossierTemplateId()) + .build(); + + RedactionResult result = redactionController.classify(redactionRequest); + + String filename = file.substring(file.lastIndexOf("/")); + + try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename)) { + fileOutputStream.write(result.getDocument()); + } + + try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename + ".json")) { + fileOutputStream.write(result.getJsonDoc()); + } } + + + } + @SneakyThrows + private Set getFileNames(Set fileNames, Path dir, String resourcePath) { + + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path path : stream) { + if (path.toFile().isDirectory()) { + getFileNames(fileNames, path, resourcePath); + } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) { + String absolutePath = cleanPath(path.toAbsolutePath().toString()); + fileNames.add(absolutePath); + } + } + } + return fileNames; + } + + private String cleanPath(String path) { + + return StringUtils.replace(path, "\\", "/"); + } + + + + @Test public void classificationTestWithCvTableService() throws IOException { @@ -1793,10 +1839,11 @@ public class RedactionIntegrationTest { @SneakyThrows private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) { - ClassPathResource pdfFileResource = new ClassPathResource(file); + File initialFile = new File(file); + InputStream targetStream = new FileInputStream(initialFile); ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile); - return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream()); + return prepareStorage(targetStream, cvServiceResponseFileResource.getInputStream()); } }