RSS-83: Imporved section detection

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -1,15 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;
 
 import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 
 import lombok.RequiredArgsConstructor;
@@ -52,11 +56,27 @@ public class ClassificationService {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
+        var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{1,2}\\s[A-Za-z]{3,50}", true);
+        var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
+
+        Matcher matcher = pattern.matcher(textBlock.toString());
+        Matcher matcher2 = pattern2.matcher(textBlock.toString());
+
         if (document.getFontSizeCounter().getMostPopular() == null) {
             textBlock.setClassification("Other");
             return;
         }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        if (textBlock.getText().length() > 5 && textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 5.9
+
+
+                && ((textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)))
+                || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT))
+                || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE")) && !textBlock.toString().endsWith(":")) {
+            textBlock.setClassification("H 1");
+            document.setHeadlines(true);
+
+        }
+        else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                 .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
             textBlock.setClassification("Header");
 
@@ -69,32 +89,39 @@ public class ClassificationService {
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                 textBlock.setClassification("Title");
             }
-        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
+        }
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
+        else if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) {
-                .getCountPerValue()
+            textBlock.setClassification("H 2");
-                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-
-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                    textBlock.setClassification("H " + i);
             document.setHeadlines(true);
         }
-            }
+//        else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+//                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
+//                .getCountPerValue()
-                .getMostPopular()
+//                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+//                .get(0)
-                .get(0)
+//                .getTextPositions()
-                .getTextPositions()
+//                .get(0)
-                .get(0)
+//                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//
-            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
+//            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-            document.setHeadlines(true);
+//                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+//                    textBlock.setClassification("H " + i);
+//                    document.setHeadlines(true);
+//                }
+//            }
+//        }
+//        else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+//                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
+//                .getMostPopular()
+//                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+//                .get(0)
+//                .getTextPositions()
+//                .get(0)
+//                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
+//            document.setHeadlines(true);
+//        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
                 .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
             textBlock.setClassification("TextBlock Bold");
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -68,7 +68,7 @@ public class SectionsBuilderService {
                     continue;
                 }
 
-                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
+                if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
                     Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -4,6 +4,7 @@ import static org.mockito.Mockito.when;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -51,9 +52,14 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.ChangeType;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
@@ -61,6 +67,7 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -133,6 +140,37 @@ public class HeadlinesGoldStandardIntegrationTest {
     private final static String TEST_FILE_ID = "123";
 
 
+    @Test
+    public void extractHeadlines() throws IOException {
+
+        AnalyzeRequest request = prepareStorage("files/RSS/31 - CA6375 - Acute Oral Toxicity - Rats.pdf");
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Headlines.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+
+        RedactionRequest redactionRequest = RedactionRequest.builder()
+                .dossierId(request.getDossierId())
+                .fileId(request.getFileId())
+                .dossierTemplateId(request.getDossierTemplateId())
+                .build();
+
+        RedactionResult result1 = redactionController.classify(redactionRequest);
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
+            fileOutputStream.write(result1.getDocument());
+        }
+    }
+
+
     @Test
     public void testHeadlineDetection() {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -364,7 +364,7 @@ public class RedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/RSS/06 - Isopyrazam - Acute Oral Toxicity Rat.pdf");
+        AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1111,7 +1111,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
+        AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())