Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RSS-83: Test new rules for headline detection 3

Browse Source
This commit is contained in:
deiflaender 2022-09-30 13:58:29 +02:00
parent 7dae9e8f0b
commit 5503c7929b
2 changed files with 26 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java
View File

@ -84,10 +84,10 @@ public class ClassificationService {
// textBlock.setClassification("Title"); // textBlock.setClassification("Title");
// } // }
} else } else
if (textBlock.getText().length() > 5 && textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 2.9 if (textBlock.getText().length() > 5 && textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 5.9
&& !textBlock.getOrientation().equals(Orientation.RIGHT) && !textBlock.getOrientation().equals(Orientation.RIGHT)
&& (textBlock.getMostPopularWordStyle().contains("bold") || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)))) { && ((textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0))) || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)))) {
textBlock.setClassification("H 1"); textBlock.setClassification("H 1");
document.setHeadlines(true); document.setHeadlines(true);

43
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java
View File

@ -103,7 +103,7 @@ public class PdfSegmentationService {
// var sorted = stripper.getTextPositionSequences().stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList()); // var sorted = stripper.getTextPositionSequences().stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
var pattern = Patterns.getCompiledPattern("^\\d[\\d.]{0,5}", true);
@ -114,24 +114,6 @@ public class PdfSegmentationService {
page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinY(), Comparator.reverseOrder()));
AbstractTextContainer prev = null;
List<AbstractTextContainer> toRemove = new ArrayList<>();
for (AbstractTextContainer current : page.getTextBlocks()) {
if(prev != null && prev instanceof TextBlock && current instanceof TextBlock){
Matcher matcher = pattern.matcher(prev.getText());
if (matcher.matches()) {
((TextBlock) current).getSequences().add(0, ((TextBlock) prev).getSequences().get(0));
toRemove.add(prev);
current.setMinX(prev.getMinX());
}
}
prev = current;
}
page.getTextBlocks().removeAll(toRemove);
// page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinX())); // page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinX()));
@ -153,11 +135,34 @@ public class PdfSegmentationService {
imageService.findOcr(page); imageService.findOcr(page);
} }
var pattern = Patterns.getCompiledPattern("^\\d[\\d.]{0,5}", true);
page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinY(), Comparator.reverseOrder()));
AbstractTextContainer prev = null;
List<AbstractTextContainer> toRemove = new ArrayList<>();
for (AbstractTextContainer current : page.getTextBlocks()) {
if(prev != null && prev instanceof TextBlock && current instanceof TextBlock && Math.abs(prev.getMinY() - current.getMinY()) <=1){
Matcher matcher = pattern.matcher(prev.getText());
if (matcher.matches()) {
((TextBlock) current).getSequences().add(0, ((TextBlock) prev).getSequences().get(0));
toRemove.add(prev);
current.setMinX(prev.getMinX());
}
}
prev = current;
}
page.getTextBlocks().removeAll(toRemove);
pages.add(page); pages.add(page);
} }
document.setPages(pages); document.setPages(pages);
classificationService.classifyDocument(document); classificationService.classifyDocument(document);
sectionsBuilderService.buildSections(document); sectionsBuilderService.buildSections(document);
sectionsBuilderService.addImagesToSections(document); sectionsBuilderService.addImagesToSections(document);