hotfix: Document parsing

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java

@@ -12,6 +12,7 @@ import lombok.NoArgsConstructor;
 public class RedactionResult {
 
     private byte[] document;
+    private byte[] jsonDoc;
     private int numberOfPages;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.controller;
 import java.io.ByteArrayOutputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.stream.Collectors;
 
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -72,29 +73,55 @@ public class RedactionController implements RedactionResource {
                         var t = page.getTextBlocks().get(i);
 
 
+                        if(t instanceof TextBlock){
+
+                            var textBlock = (TextBlock) t;
+
                             document.getParagraphs()
                                     .add(Paragraph.builder()
-                                        .text(t.getText())
+                                            .text(textBlock.getText())
-                                        .boundingBox(new Rectangle(new Point(t.getMinX(), t.getMinY()), t.getWidth(), t.getHeight(), t.getPage()))
+                                            .boundingBox(new Rectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock.getHeight(), textBlock.getPage()))
-                                        .textStyle(((TextBlock) t).getMostPopularWordStyle())
+                                            .textStyle(textBlock.getMostPopularWordStyle())
                                             .paragraphNum(i)
-                                        .orientation(t.getOrientation())
+                                            .orientation(textBlock.getOrientation())
-                                        .classification(t.getClassification())
+                                            .classification(textBlock.getClassification())
-                                        .textDirection(((TextBlock) t).getSequences().get(0).getDir())
+                                            .textDirection(textBlock.getSequences().get(0).getDir())
-                                        .page(t.getPage())
+                                            .page(textBlock.getPage())
                                             .build());
+                        } else {
+
+                            var table = (Table) t;
+
+                            for (var row: table.getRows()){
+
+                                StringBuilder sb = new StringBuilder();
+
+                                row.forEach(cell -> {
+                                    sb.append(cell.getTextBlocks().stream().map(a -> a.getText()).collect(Collectors.joining("|")));
+                                });
+
+                                document.getParagraphs()
+                                        .add(Paragraph.builder()
+                                                .text(sb.toString())
+                                                .boundingBox(new Rectangle(new Point(table.getMinX(), table.getMinY()), table.getWidth(), table.getHeight(), table.getPage()))
+                                                .paragraphNum(i)
+                                                .orientation(table.getOrientation())
+                                                .classification("Table Row")
+                                                .page(table.getPage())
+                                                .build());
+                            }
+                        }
+
+
 
                     }
 
                 });
 
-                try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Document.json")) {
-                    fileOutputStream.write(objectMapper.writeValueAsBytes(document));
-                }
 
                 pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
 
-                return convert(pdDocument, classifiedDoc.getPages().size());
+                return convert(pdDocument, classifiedDoc.getPages().size(), objectMapper.writeValueAsBytes(document));
 
             } catch (IOException e) {
                 throw new RedactionException(e);
@@ -123,7 +150,7 @@ public class RedactionController implements RedactionResource {
                 pdDocument.setAllSecurityToBeRemoved(true);
 
                 pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
-                return convert(pdDocument, classifiedDoc.getPages().size());
+                return convert(pdDocument, classifiedDoc.getPages().size(), null);
 
             } catch (IOException e) {
                 throw new RedactionException(e);
@@ -183,11 +210,12 @@ public class RedactionController implements RedactionResource {
     }
 
 
-    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
+    private RedactionResult convert(PDDocument document, int numberOfPages, byte[] json) throws IOException {
 
         try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
             document.save(byteArrayOutputStream);
-            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
+            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages)
+                    .jsonDoc(json).build();
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -6,12 +6,17 @@ import static org.mockito.Mockito.when;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
@@ -24,6 +29,7 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -1111,7 +1117,13 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/new/2001160.pdf");
+        String sourcePath ="/tmp";
+        String targetPath = "/tmp/result";
+
+        var files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(sourcePath), sourcePath);
+
+        for(var file : files){
+            AnalyzeRequest request = prepareStorage(file);
 
             RedactionRequest redactionRequest = RedactionRequest.builder()
                     .dossierId(request.getDossierId())
@@ -1121,10 +1133,44 @@ public class RedactionIntegrationTest {
 
             RedactionResult result = redactionController.classify(redactionRequest);
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
+            String filename = file.substring(file.lastIndexOf("/"));
+
+            try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename)) {
                 fileOutputStream.write(result.getDocument());
             }
+
+            try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename + ".json")) {
+                fileOutputStream.write(result.getJsonDoc());
             }
+        }
+
+
+
+    }
+
+
+    @SneakyThrows
+    private Set<String> getFileNames(Set<String> fileNames, Path dir, String resourcePath) {
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
+            for (Path path : stream) {
+                if (path.toFile().isDirectory()) {
+                    getFileNames(fileNames, path, resourcePath);
+                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
+                    String absolutePath = cleanPath(path.toAbsolutePath().toString());
+                    fileNames.add(absolutePath);
+                }
+            }
+        }
+        return fileNames;
+    }
+
+    private String cleanPath(String path) {
+
+        return StringUtils.replace(path, "\\", "/");
+    }
+
+
 
 
     @Test
@@ -1793,10 +1839,11 @@ public class RedactionIntegrationTest {
     @SneakyThrows
     private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        File initialFile = new File(file);
+        InputStream targetStream = new FileInputStream(initialFile);
         ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
 
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+        return prepareStorage(targetStream, cvServiceResponseFileResource.getInputStream());
     }
 
 }