Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

DM-307 Improved headline detection #56

Merged
dominique.eiflaender1 merged 1 commits from DM-307 into master 2023-07-18 15:29:22 +02:00
Conversation 3 Commits 1 Files Changed 2 +8 -7
2 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/DocuMineClassificationService.java
View File

@ -59,8 +59,8 @@ public class DocuMineClassificationService implements ClassificationService {
log.debug("headlineFontSizes: {}", headlineFontSizes);
var bodyTextFrame = page.getBodyTextFrame();
var pattern = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z]{2,50}", true);
var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
var pattern = Patterns.getCompiledPattern("^(\\d{1,1}\\.?){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z\\[\\]\\-]{2,50}", true);
var pattern2 = Patterns.getCompiledPattern(".*\\d{4}$", true);
var pattern3 = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*", false);
Matcher matcher = pattern.matcher(textBlock.toString());
@ -71,16 +71,16 @@ public class DocuMineClassificationService implements ClassificationService {
textBlock.setClassification(PageBlockType.OTHER);
return;
}
if (textBlock.getText().length() > 5 && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter()
if (textBlock.getText().length() > 6 && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter()
.getMostPopular() || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()) && PositionUtils.getApproxLineCount(textBlock) < 5.9
&& (textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && !matcher2.matches() && !textBlock.toString()
.contains(":") || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && !matcher2.matches() && !textBlock.toString()
.contains(":") || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE")) && !textBlock.toString().endsWith(":")) {
.contains(":") || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("FIGURE") || textBlock.toString().startsWith("TABLE")) && !textBlock.toString().endsWith(":")) {
textBlock.setClassification(PageBlockType.getHeadlineType(1));
document.setHeadlines(true);
} else if (matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.matches() && !matcher3.matches()) {
} else if (matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher3.matches() && !matcher2.matches()) {
textBlock.setClassification(PageBlockType.getHeadlineType(2));
document.setHeadlines(true);
} else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java
View File

@ -82,7 +82,8 @@ public class SectionNodeFactory {
if (abstractPageBlock instanceof TextPageBlock) {
// List<TextPageBlock> textBlocksToMerge = findTextBlocksWithSameClassificationAndAlignsYAndSameOrientationUntilConvergence((TextPageBlock) abstractPageBlock, remainingBlocks);
List<TextPageBlock> textBlocksToMerge = findTextBlocksWithSameClassificationAndAlignsYAndSameOrientation(List.of((TextPageBlock) abstractPageBlock), remainingBlocks);
List<TextPageBlock> textBlocksToMerge = findTextBlocksWithSameClassificationAndAlignsYAndSameOrientation(List.of((TextPageBlock) abstractPageBlock),
remainingBlocks);
alreadyMerged.addAll(textBlocksToMerge);
DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocksToMerge);
} else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
@ -125,7 +126,7 @@ public class SectionNodeFactory {
List<AbstractPageBlock> previousList = splitList.get(i - 1);
AbstractPageBlock lastPageBlockInPreviousList = previousList.get(previousList.size() - 1);
if (lastPageBlockInPreviousList.isHeadline()) {
previousList.remove(i - 1);
previousList.remove(previousList.size() - 1);
splitList.get(i).add(0, lastPageBlockInPreviousList);
}
}