hotfix: Document parsing

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java

@@ -12,6 +12,7 @@ import lombok.NoArgsConstructor;
 public class RedactionResult {
 
     private byte[] document;
+    private byte[] jsonDoc;
     private int numberOfPages;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java

@@ -84,11 +84,11 @@ public class BodyTextFrameService {
                     }
 
                     float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (approxLineCount < 2.9f) {
+                    if (approxLineCount < 3.3f) {
                         continue;
                     }
 
-                    if (documentFontSizeCounter.getMostPopular() != null && textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
+                    if (documentFontSizeCounter.getMostPopular() != null ) {
 
                         expandRectangle(textBlock, page, expansionsRectangle);
                     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -56,8 +56,7 @@ public class ClassificationService {
             textBlock.setClassification("Other");
             return;
         }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) ) {
             textBlock.setClassification("Header");
 
         } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Document.java

@@ -0,0 +1,13 @@
+package com.iqser.red.service.redaction.v1.server.controller;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class Document {
+
+    private List<Paragraph> paragraphs = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Paragraph.java

@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.server.controller;
+
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Builder
+@Data
+public class Paragraph {
+        private int paragraphNum;
+        private String text;
+        private String textStyle;
+        private String classification;
+        private int page;
+        private Rectangle boundingBox;
+        private Orientation orientation;
+        private TextDirection textDirection;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,21 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
 import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.stream.Collectors;
 
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
@@ -41,6 +47,7 @@ public class RedactionController implements RedactionResource {
     private final RedactionStorageService redactionStorageService;
     private final RedactionLogMergeService redactionLogMergeService;
     private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
+    private final ObjectMapper objectMapper;
 
 
     @Override
@@ -58,9 +65,63 @@ public class RedactionController implements RedactionResource {
             try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
                 pdDocument.setAllSecurityToBeRemoved(true);
 
+                com.iqser.red.service.redaction.v1.server.controller.Document document = new com.iqser.red.service.redaction.v1.server.controller.Document();
+
+                classifiedDoc.getPages().forEach(page -> {
+
+                    for (int i = 0; i < page.getTextBlocks().size(); i++) {
+                        var t = page.getTextBlocks().get(i);
+
+
+                        if(t instanceof TextBlock){
+
+                            var textBlock = (TextBlock) t;
+
+                            document.getParagraphs()
+                                    .add(Paragraph.builder()
+                                            .text(textBlock.getText())
+                                            .boundingBox(new Rectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock.getHeight(), textBlock.getPage()))
+                                            .textStyle(textBlock.getMostPopularWordStyle())
+                                            .paragraphNum(i)
+                                            .orientation(textBlock.getOrientation())
+                                            .classification(textBlock.getClassification())
+                                            .textDirection(textBlock.getSequences().get(0).getDir())
+                                            .page(textBlock.getPage())
+                                            .build());
+                        } else {
+
+                            var table = (Table) t;
+
+                            for (var row: table.getRows()){
+
+                                StringBuilder sb = new StringBuilder();
+
+                                row.forEach(cell -> {
+                                    sb.append(cell.getTextBlocks().stream().map(a -> a.getText()).collect(Collectors.joining("|")));
+                                });
+
+                                document.getParagraphs()
+                                        .add(Paragraph.builder()
+                                                .text(sb.toString())
+                                                .boundingBox(new Rectangle(new Point(table.getMinX(), table.getMinY()), table.getWidth(), table.getHeight(), table.getPage()))
+                                                .paragraphNum(i)
+                                                .orientation(table.getOrientation())
+                                                .classification("Table Row")
+                                                .page(table.getPage())
+                                                .build());
+                            }
+                        }
+
+
+
+                    }
+
+                });
+
+
                 pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
 
-                return convert(pdDocument, classifiedDoc.getPages().size());
+                return convert(pdDocument, classifiedDoc.getPages().size(), objectMapper.writeValueAsBytes(document));
 
             } catch (IOException e) {
                 throw new RedactionException(e);
@@ -89,7 +150,7 @@ public class RedactionController implements RedactionResource {
                 pdDocument.setAllSecurityToBeRemoved(true);
 
                 pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
-                return convert(pdDocument, classifiedDoc.getPages().size());
+                return convert(pdDocument, classifiedDoc.getPages().size(), null);
 
             } catch (IOException e) {
                 throw new RedactionException(e);
@@ -149,11 +210,12 @@ public class RedactionController implements RedactionResource {
     }
 
 
-    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
+    private RedactionResult convert(PDDocument document, int numberOfPages, byte[] json) throws IOException {
 
         try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
             document.save(byteArrayOutputStream);
-            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
+            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages)
+                    .jsonDoc(json).build();
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -131,8 +131,8 @@ public class PdfSegmentationService {
                 document.setPages(pages);
 
                 classificationService.classifyDocument(document);
-                sectionsBuilderService.buildSections(document);
-                sectionsBuilderService.addImagesToSections(document);
+//                sectionsBuilderService.buildSections(document);
+//                sectionsBuilderService.addImagesToSections(document);
 
                 IOUtils.close(pdDocument);
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -6,12 +6,17 @@ import static org.mockito.Mockito.when;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
@@ -24,6 +29,7 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -1111,7 +1117,13 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
+        String sourcePath ="/tmp";
+        String targetPath = "/tmp/result";
+
+        var files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(sourcePath), sourcePath);
+
+        for(var file : files){
+            AnalyzeRequest request = prepareStorage(file);
 
             RedactionRequest redactionRequest = RedactionRequest.builder()
                     .dossierId(request.getDossierId())
@@ -1121,10 +1133,44 @@ public class RedactionIntegrationTest {
 
             RedactionResult result = redactionController.classify(redactionRequest);
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
+            String filename = file.substring(file.lastIndexOf("/"));
+
+            try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename)) {
                 fileOutputStream.write(result.getDocument());
             }
+
+            try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename + ".json")) {
+                fileOutputStream.write(result.getJsonDoc());
             }
+        }
+
+
+
+    }
+
+
+    @SneakyThrows
+    private Set<String> getFileNames(Set<String> fileNames, Path dir, String resourcePath) {
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
+            for (Path path : stream) {
+                if (path.toFile().isDirectory()) {
+                    getFileNames(fileNames, path, resourcePath);
+                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
+                    String absolutePath = cleanPath(path.toAbsolutePath().toString());
+                    fileNames.add(absolutePath);
+                }
+            }
+        }
+        return fileNames;
+    }
+
+    private String cleanPath(String path) {
+
+        return StringUtils.replace(path, "\\", "/");
+    }
+
+
 
 
     @Test
@@ -1793,10 +1839,11 @@ public class RedactionIntegrationTest {
     @SneakyThrows
     private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        File initialFile = new File(file);
+        InputStream targetStream = new FileInputStream(initialFile);
         ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
 
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+        return prepareStorage(targetStream, cvServiceResponseFileResource.getInputStream());
     }
 
 }