Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RSS-83: Test new rules for headline detection 2

Browse Source
This commit is contained in:
deiflaender 2022-09-30 12:30:35 +02:00
parent fb1892ea20
commit 7dae9e8f0b
3 changed files with 48 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java
View File

@ -55,7 +55,7 @@ public class ClassificationService {
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
List<Float> headlineFontSizes) {
var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{0,2}\\s[A-Za-z]{3,50}", true);
var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{1,2}\\s[A-Za-z]{3,50}", true);
var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
Matcher matcher = pattern.matcher(textBlock.toString());
@ -91,7 +91,8 @@ public class ClassificationService {
textBlock.setClassification("H 1");
document.setHeadlines(true);
} else
}
else
if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) {
textBlock.setClassification("H 2");
document.setHeadlines(true);

34
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java
View File

@ -9,9 +9,12 @@ import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.SystemUtils;
@ -29,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.classification.service.Classifi
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
@ -97,10 +101,40 @@ public class PdfSegmentationService {
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
.getMaxCharHeight());
// var sorted = stripper.getTextPositionSequences().stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
var pattern = Patterns.getCompiledPattern("^\\d[\\d.]{0,5}", true);
Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
.getVertical());
page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinY(), Comparator.reverseOrder()));
AbstractTextContainer prev = null;
List<AbstractTextContainer> toRemove = new ArrayList<>();
for (AbstractTextContainer current : page.getTextBlocks()) {
if(prev != null && prev instanceof TextBlock && current instanceof TextBlock){
Matcher matcher = pattern.matcher(prev.getText());
if (matcher.matches()) {
((TextBlock) current).getSequences().add(0, ((TextBlock) prev).getSequences().get(0));
toRemove.add(prev);
current.setMinX(prev.getMinX());
}
}
prev = current;
}
page.getTextBlocks().removeAll(toRemove);
// page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinX()));
PDRectangle cropbox = pdPage.getCropBox();
float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
page.setCropBoxArea(cropboxArea);

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java
View File

@ -32,6 +32,7 @@ public class PdfVisualisationService {
PDPage pdPage = document.getPage(page - 1);
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
int a = 0;
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
@ -43,7 +44,8 @@ public class PdfVisualisationService {
}
if (textBlock instanceof TextBlock) {
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
visualizeTextBlock((TextBlock) textBlock, contentStream);
visualizeTextBlock((TextBlock) textBlock, contentStream, a);
a++;
} else if (textBlock instanceof Table) {
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
visualizeTable((Table) textBlock, contentStream);
@ -59,7 +61,9 @@ public class PdfVisualisationService {
public void visualizeClassifications(Document classifiedDoc, PDDocument document) throws IOException {
for (int page = 1; page <= document.getNumberOfPages(); page++) {
int a = 0;
Page analyzedPage = classifiedDoc.getPages().get(page - 1);
@ -72,12 +76,14 @@ public class PdfVisualisationService {
continue;
}
if (textBlock instanceof TextBlock) {
visualizeTextBlock((TextBlock) textBlock, contentStream);
visualizeTextBlock((TextBlock) textBlock, contentStream, a);
a++;
} else if (textBlock instanceof Table) {
visualizeTable((Table) textBlock, contentStream);
}
}
contentStream.setStrokingColor(Color.YELLOW);
contentStream.addRect((float) analyzedPage.getBodyTextFrame().getX(), (float) analyzedPage.getBodyTextFrame().getY(), (float) analyzedPage.getBodyTextFrame().getWidth(), (float) analyzedPage.getBodyTextFrame().getHeight());
contentStream.stroke();
@ -87,7 +93,7 @@ public class PdfVisualisationService {
}
private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream, int number) throws IOException {
contentStream.setStrokingColor(Color.RED);
@ -98,11 +104,11 @@ public class PdfVisualisationService {
contentStream.beginText();
contentStream.setNonStrokingColor(Color.BLUE);
contentStream.setFont(PDType1Font.TIMES_ROMAN, 12f);
contentStream.setFont(PDType1Font.TIMES_ROMAN, 6f);
contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());
contentStream.showText(textBlock.getClassification() + textBlock.getOrientation());
contentStream.showText(textBlock.getClassification() /*" textBlock.getOrientation() + "--> "*/ + number);
contentStream.endText();
}