diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java index 63a24e77..65490e47 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java @@ -55,7 +55,7 @@ public class ClassificationService { public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List headlineFontSizes) { - var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{0,2}\\s[A-Za-z]{3,50}", true); + var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{1,2}\\s[A-Za-z]{3,50}", true); var pattern2 = Patterns.getCompiledPattern(".*\\d$", true); Matcher matcher = pattern.matcher(textBlock.toString()); @@ -91,7 +91,8 @@ public class ClassificationService { textBlock.setClassification("H 1"); document.setHeadlines(true); - } else + } + else if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) { textBlock.setClassification("H 2"); document.setHeadlines(true); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java index efbf1b5e..7f72bee2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java @@ -9,9 +9,12 @@ import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.SystemUtils; @@ -29,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.classification.service.Classifi import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; +import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings; import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService; @@ -97,10 +101,40 @@ public class PdfSegmentationService { CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper .getMaxCharHeight()); +// var sorted = stripper.getTextPositionSequences().stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList()); + + var pattern = Patterns.getCompiledPattern("^\\d[\\d.]{0,5}", true); + + + + Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings .getVertical()); + + + page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinY(), Comparator.reverseOrder())); + + AbstractTextContainer prev = null; + List toRemove = new ArrayList<>(); + for (AbstractTextContainer current : page.getTextBlocks()) { + + if(prev != null && prev instanceof TextBlock && current instanceof TextBlock){ + Matcher matcher = pattern.matcher(prev.getText()); + if (matcher.matches()) { + ((TextBlock) current).getSequences().add(0, ((TextBlock) prev).getSequences().get(0)); + toRemove.add(prev); + current.setMinX(prev.getMinX()); + } + } + prev = current; + } + + page.getTextBlocks().removeAll(toRemove); + +// page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinX())); + PDRectangle cropbox = pdPage.getCropBox(); float cropboxArea = cropbox.getHeight() * cropbox.getWidth(); page.setCropBoxArea(cropboxArea); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java index 390810a4..0b0f121f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java @@ -32,6 +32,7 @@ public class PdfVisualisationService { PDPage pdPage = document.getPage(page - 1); PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true); + int a = 0; for (Paragraph paragraph : classifiedDoc.getParagraphs()) { for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) { @@ -43,7 +44,8 @@ public class PdfVisualisationService { } if (textBlock instanceof TextBlock) { textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size()); - visualizeTextBlock((TextBlock) textBlock, contentStream); + visualizeTextBlock((TextBlock) textBlock, contentStream, a); + a++; } else if (textBlock instanceof Table) { textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size()); visualizeTable((Table) textBlock, contentStream); @@ -59,7 +61,9 @@ public class PdfVisualisationService { public void visualizeClassifications(Document classifiedDoc, PDDocument document) throws IOException { + for (int page = 1; page <= document.getNumberOfPages(); page++) { + int a = 0; Page analyzedPage = classifiedDoc.getPages().get(page - 1); @@ -72,12 +76,14 @@ public class PdfVisualisationService { continue; } if (textBlock instanceof TextBlock) { - visualizeTextBlock((TextBlock) textBlock, contentStream); + visualizeTextBlock((TextBlock) textBlock, contentStream, a); + a++; } else if (textBlock instanceof Table) { visualizeTable((Table) textBlock, contentStream); } } + contentStream.setStrokingColor(Color.YELLOW); contentStream.addRect((float) analyzedPage.getBodyTextFrame().getX(), (float) analyzedPage.getBodyTextFrame().getY(), (float) analyzedPage.getBodyTextFrame().getWidth(), (float) analyzedPage.getBodyTextFrame().getHeight()); contentStream.stroke(); @@ -87,7 +93,7 @@ public class PdfVisualisationService { } - private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException { + private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream, int number) throws IOException { contentStream.setStrokingColor(Color.RED); @@ -98,11 +104,11 @@ public class PdfVisualisationService { contentStream.beginText(); contentStream.setNonStrokingColor(Color.BLUE); - contentStream.setFont(PDType1Font.TIMES_ROMAN, 12f); + contentStream.setFont(PDType1Font.TIMES_ROMAN, 6f); contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY()); - contentStream.showText(textBlock.getClassification() + textBlock.getOrientation()); + contentStream.showText(textBlock.getClassification() /*" textBlock.getOrientation() + "--> "*/ + number); contentStream.endText(); }