RSS-83: Imporved section detection 2

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -54,8 +54,9 @@ public class BlockificationService {
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
             boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
+            boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
 
-            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
+            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {
@@ -107,51 +108,51 @@ public class BlockificationService {
             chunkBlockList1.add(cb1);
         }
 
-        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
-
-        TextBlock previousLeft = null;
-        TextBlock previousRight = null;
-        while (itty.hasNext()) {
-            TextBlock block = (TextBlock) itty.next();
-
-            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
-                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
-                    previousLeft.add(block);
-                    itty.remove();
-                    continue;
-                }
-            }
-
-            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
-                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
-                    previousRight.add(block);
-                    itty.remove();
-                    continue;
-                }
-            }
-
-            if (block.getOrientation().equals(Orientation.LEFT)) {
-                previousLeft = block;
-            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
-                previousRight = block;
-            }
-        }
-
-        itty = chunkBlockList1.iterator();
-        TextBlock previous = null;
-        while (itty.hasNext()) {
-            TextBlock block = (TextBlock) itty.next();
-
-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
-                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
-                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
-                previous.add(block);
-                itty.remove();
-                continue;
-            }
-
-            previous = block;
-        }
+//        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
+//
+//        TextBlock previousLeft = null;
+//        TextBlock previousRight = null;
+//        while (itty.hasNext()) {
+//            TextBlock block = (TextBlock) itty.next();
+//
+//            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
+//                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
+//                    previousLeft.add(block);
+//                    itty.remove();
+//                    continue;
+//                }
+//            }
+//
+//            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
+//                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
+//                    previousRight.add(block);
+//                    itty.remove();
+//                    continue;
+//                }
+//            }
+//
+//            if (block.getOrientation().equals(Orientation.LEFT)) {
+//                previousLeft = block;
+//            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
+//                previousRight = block;
+//            }
+//        }
+//
+//        itty = chunkBlockList1.iterator();
+//        TextBlock previous = null;
+//        while (itty.hasNext()) {
+//            TextBlock block = (TextBlock) itty.next();
+//
+//            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
+//                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
+//                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
+//                previous.add(block);
+//                itty.remove();
+//                continue;
+//            }
+//
+//            previous = block;
+//        }
 
         return new Page(chunkBlockList1);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -56,26 +56,31 @@ public class ClassificationService {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
-        var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{1,2}\\s[A-Za-z]{3,50}", true);
+        var pattern = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z]{2,50}", true);
         var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
+        var pattern3 = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*", false);
 
         Matcher matcher = pattern.matcher(textBlock.toString());
         Matcher matcher2 = pattern2.matcher(textBlock.toString());
+        Matcher matcher3 = pattern3.matcher(textBlock.toString());
 
         if (document.getFontSizeCounter().getMostPopular() == null) {
             textBlock.setClassification("Other");
             return;
         }
-        if (textBlock.getText().length() > 5 && textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 5.9
+        if (textBlock.getText().length() > 5 && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()) && PositionUtils.getApproxLineCount(textBlock) < 5.9
 
-
-                && ((textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)))
-                || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT))
-                || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE")) && !textBlock.toString().endsWith(":")) {
+                && ((textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && !matcher2.matches() && !textBlock.toString().contains(":")
+                || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && !matcher2.matches() && !textBlock.toString().contains(":")
+                || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE")) && !textBlock.toString().endsWith(":"))) {
             textBlock.setClassification("H 1");
             document.setHeadlines(true);
 
         }
+        else if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.matches() && !matcher3.matches()) {
+            textBlock.setClassification("H 2");
+            document.setHeadlines(true);
+        }
         else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                 .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
             textBlock.setClassification("Header");
@@ -90,10 +95,7 @@ public class ClassificationService {
                 textBlock.setClassification("Title");
             }
         }
-        else if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) {
-            textBlock.setClassification("H 2");
-            document.setHeadlines(true);
-        }
+
 //        else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
 //                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
 //                .getCountPerValue()

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -143,7 +143,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     @Test
     public void extractHeadlines() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/RSS/31 - CA6375 - Acute Oral Toxicity - Rats.pdf");
+        AnalyzeRequest request = prepareStorage("files/RSS/26 - Sedaxane - Acute Oral Toxicity - Rat.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         analyzeService.analyze(request);
@@ -165,7 +165,7 @@ public class HeadlinesGoldStandardIntegrationTest {
 
         RedactionResult result1 = redactionController.classify(redactionRequest);
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/HeadlinesClassified.pdf")) {
             fileOutputStream.write(result1.getDocument());
         }
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -1098,7 +1098,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
+        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())