diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java index 95eb4e1e..1dc54ae2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java @@ -84,11 +84,11 @@ public class BodyTextFrameService { } float approxLineCount = PositionUtils.getApproxLineCount(textBlock); - if (approxLineCount < 2.9f) { + if (approxLineCount < 3.3f) { continue; } - if (documentFontSizeCounter.getMostPopular() != null && textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) { + if (documentFontSizeCounter.getMostPopular() != null ) { expandRectangle(textBlock, page, expansionsRectangle); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java index 35381230..a7773af8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java @@ -56,8 +56,7 @@ public class ClassificationService { textBlock.setClassification("Other"); return; } - if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter() - .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) { + if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) ) { textBlock.setClassification("Header"); } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Document.java new file mode 100644 index 00000000..b662373b --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Document.java @@ -0,0 +1,13 @@ +package com.iqser.red.service.redaction.v1.server.controller; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Data; + +@Data +public class Document { + + private List paragraphs = new ArrayList<>(); + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Paragraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Paragraph.java new file mode 100644 index 00000000..4d12f7fb --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Paragraph.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.redaction.v1.server.controller; + +import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.redaction.v1.server.classification.model.Orientation; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection; + +import lombok.Builder; +import lombok.Data; + +@Builder +@Data +public class Paragraph { + private int paragraphNum; + private String text; + private String textStyle; + private String classification; + private int page; + private Rectangle boundingBox; + private Orientation orientation; + private TextDirection textDirection; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 96491cfc..b5022156 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.controller; import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; import java.io.IOException; import org.apache.pdfbox.pdmodel.PDDocument; @@ -8,14 +9,18 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; +import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.redaction.v1.model.Point; +import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionRequest; import com.iqser.red.service.redaction.v1.model.RedactionResult; import com.iqser.red.service.redaction.v1.resources.RedactionResource; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; +import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; @@ -41,6 +46,7 @@ public class RedactionController implements RedactionResource { private final RedactionStorageService redactionStorageService; private final RedactionLogMergeService redactionLogMergeService; private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService; + private final ObjectMapper objectMapper; @Override @@ -58,6 +64,34 @@ public class RedactionController implements RedactionResource { try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) { pdDocument.setAllSecurityToBeRemoved(true); + com.iqser.red.service.redaction.v1.server.controller.Document document = new com.iqser.red.service.redaction.v1.server.controller.Document(); + + classifiedDoc.getPages().forEach(page -> { + + for (int i = 0; i < page.getTextBlocks().size(); i++) { + var t = page.getTextBlocks().get(i); + + + document.getParagraphs() + .add(Paragraph.builder() + .text(t.getText()) + .boundingBox(new Rectangle(new Point(t.getMinX(), t.getMinY()), t.getWidth(), t.getHeight(), t.getPage())) + .textStyle(((TextBlock) t).getMostPopularWordStyle()) + .paragraphNum(i) + .orientation(t.getOrientation()) + .classification(t.getClassification()) + .textDirection(((TextBlock) t).getSequences().get(0).getDir()) + .page(t.getPage()) + .build()); + + } + + }); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Document.json")) { + fileOutputStream.write(objectMapper.writeValueAsBytes(document)); + } + pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument); return convert(pdDocument, classifiedDoc.getPages().size()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java index 05166420..e96a637d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java @@ -131,8 +131,8 @@ public class PdfSegmentationService { document.setPages(pages); classificationService.classifyDocument(document); - sectionsBuilderService.buildSections(document); - sectionsBuilderService.addImagesToSections(document); +// sectionsBuilderService.buildSections(document); +// sectionsBuilderService.addImagesToSections(document); IOUtils.close(pdDocument); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 38360d8b..9a0399eb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1111,7 +1111,7 @@ public class RedactionIntegrationTest { System.out.println("classificationTest"); - AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf"); + AnalyzeRequest request = prepareStorage("files/new/2001160.pdf"); RedactionRequest redactionRequest = RedactionRequest.builder() .dossierId(request.getDossierId()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/2001160.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/2001160.pdf new file mode 100644 index 00000000..f27b517d Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/2001160.pdf differ