RSS-83: Test new rules for headline detection 2

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -55,7 +55,7 @@ public class ClassificationService {
     public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
                               List<Float> headlineFontSizes) {
 
-        var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{0,2}\\s[A-Za-z]{3,50}", true);
+        var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{1,2}\\s[A-Za-z]{3,50}", true);
         var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
 
         Matcher matcher = pattern.matcher(textBlock.toString());
@@ -91,7 +91,8 @@ public class ClassificationService {
                     textBlock.setClassification("H 1");
                     document.setHeadlines(true);
 
-        } else
+        }
+            else
              if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) {
                 textBlock.setClassification("H 2");
                 document.setHeadlines(true);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -9,9 +9,12 @@ import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.SystemUtils;
@@ -29,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.classification.service.Classifi
 import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
@@ -97,10 +101,40 @@ public class PdfSegmentationService {
                     CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
                             .getMaxCharHeight());
 
+//                    var sorted = stripper.getTextPositionSequences().stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
+
+                    var pattern = Patterns.getCompiledPattern("^\\d[\\d.]{0,5}", true);
+
+
+
+
                     Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
                             .getVertical());
 
 
+
+
+                    page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinY(), Comparator.reverseOrder()));
+
+                    AbstractTextContainer prev = null;
+                    List<AbstractTextContainer> toRemove = new ArrayList<>();
+                    for (AbstractTextContainer current : page.getTextBlocks()) {
+
+                        if(prev != null && prev instanceof TextBlock && current instanceof TextBlock){
+                            Matcher matcher = pattern.matcher(prev.getText());
+                            if (matcher.matches()) {
+                                ((TextBlock) current).getSequences().add(0, ((TextBlock) prev).getSequences().get(0));
+                                toRemove.add(prev);
+                                current.setMinX(prev.getMinX());
+                            }
+                        }
+                        prev = current;
+                    }
+
+                    page.getTextBlocks().removeAll(toRemove);
+
+//                    page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinX()));
+
                     PDRectangle cropbox = pdPage.getCropBox();
                     float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
                     page.setCropBoxArea(cropboxArea);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -32,6 +32,7 @@ public class PdfVisualisationService {
             PDPage pdPage = document.getPage(page - 1);
             PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
 
+            int a = 0;
             for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
 
                 for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
@@ -43,7 +44,8 @@ public class PdfVisualisationService {
                     }
                     if (textBlock instanceof TextBlock) {
                         textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
-                        visualizeTextBlock((TextBlock) textBlock, contentStream);
+                        visualizeTextBlock((TextBlock) textBlock, contentStream, a);
+                        a++;
                     } else if (textBlock instanceof Table) {
                         textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
                         visualizeTable((Table) textBlock, contentStream);
@@ -59,7 +61,9 @@ public class PdfVisualisationService {
 
     public void visualizeClassifications(Document classifiedDoc, PDDocument document) throws IOException {
 
+
         for (int page = 1; page <= document.getNumberOfPages(); page++) {
+            int a = 0;
 
             Page analyzedPage = classifiedDoc.getPages().get(page - 1);
 
@@ -72,12 +76,14 @@ public class PdfVisualisationService {
                     continue;
                 }
                 if (textBlock instanceof TextBlock) {
-                    visualizeTextBlock((TextBlock) textBlock, contentStream);
+                    visualizeTextBlock((TextBlock) textBlock, contentStream, a);
+                    a++;
                 } else if (textBlock instanceof Table) {
                     visualizeTable((Table) textBlock, contentStream);
                 }
             }
 
+
             contentStream.setStrokingColor(Color.YELLOW);
             contentStream.addRect((float) analyzedPage.getBodyTextFrame().getX(), (float) analyzedPage.getBodyTextFrame().getY(), (float) analyzedPage.getBodyTextFrame().getWidth(), (float) analyzedPage.getBodyTextFrame().getHeight());
             contentStream.stroke();
@@ -87,7 +93,7 @@ public class PdfVisualisationService {
     }
 
 
-    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
+    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream, int number) throws IOException {
 
         contentStream.setStrokingColor(Color.RED);
 
@@ -98,11 +104,11 @@ public class PdfVisualisationService {
             contentStream.beginText();
 
             contentStream.setNonStrokingColor(Color.BLUE);
-            contentStream.setFont(PDType1Font.TIMES_ROMAN, 12f);
+            contentStream.setFont(PDType1Font.TIMES_ROMAN, 6f);
 
             contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());
 
-            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation());
+            contentStream.showText(textBlock.getClassification() /*" textBlock.getOrientation() + "--> "*/ + number);
 
             contentStream.endText();
         }