Merge branch 'RM-231-bp-bp' into 'release/0.159.x'

RM-231: missing whitespace in name

See merge request fforesight/layout-parser!265

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -24,7 +24,7 @@ import lombok.EqualsAndHashCode;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
 public class Line extends TextBoundingBox {
 
-    private static final double WORD_DISTANCE_MULTIPLIER = 0.18;
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.17;
 
     @EqualsAndHashCode.Include
     private final double x0;
@@ -159,6 +159,9 @@ public class Line extends TextBoundingBox {
 
     private void computeWords(double wordSpacing) {
 
+        // Imo, the width of space should be scaled with the font size, but it only depends on the median distance between horizontal neighbours.
+        // If there are large differences in fontsize on a page, this might lead to missing spaces for the smaller fonts and too many for larger fonts.
+        // I've just now changed the scaling factor. If you come across this comment with missing whitespaces again, try scaling the fontsize instead of simply changing the factor again.
         Word word = new Word();
         Character previous = null;
         for (Character current : characters) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClassificationPatterns.java

@@ -18,10 +18,14 @@ public class ClassificationPatterns {
 
 
 
-    public static final Pattern TABLE_OR_FIGURE_PATTERN = Pattern.compile(
+    public static final Pattern TABLE_OR_FIGURE_HEADLINE_PATTERN = Pattern.compile(
             "^\\s*(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
             Pattern.CASE_INSENSITIVE);
 
+    public static final Pattern TABLE_MID_SENTENCE_PATTERN = Pattern.compile(
+            "(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
+            Pattern.CASE_INSENSITIVE);
+
     public static final Pattern ALPHANUMERIC = Pattern.compile("[a-zA-Z0-9]");
 
     public static final Pattern NUMERIC = Pattern.compile("[0-9]+");

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/DocuMineClassificationService.java

@@ -6,7 +6,8 @@ import static com.knecon.fforesight.service.layoutparser.processor.services.clas
 import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_PATTERN_WITH_SLASHES;
 import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_2_IDENTIFER_PATTERN;
 import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_OR_FIGURE_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_MID_SENTENCE_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_OR_FIGURE_HEADLINE_PATTERN;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -83,7 +84,8 @@ public class DocuMineClassificationService {
         Matcher atLeast3Matcher = AT_LEAST_3_CHARS_PATTERN.matcher(textBlock.toString());
         Matcher headlineWithSlashesMatcher = HEADLINE_PATTERN_WITH_SLASHES.matcher(textBlock.toString());
         Matcher amountMatcher = AMOUNT_PATTERN.matcher(textBlock.toString());
-        Matcher tableOrFigureMatcher = TABLE_OR_FIGURE_PATTERN.matcher(textBlock.toString());
+        Matcher tableOrFigureMatcher = TABLE_OR_FIGURE_HEADLINE_PATTERN.matcher(textBlock.toString());
+        Matcher tableMidSentenceMatcher = TABLE_MID_SENTENCE_PATTERN.matcher(textBlock.toString());
         Matcher headlineWithSingleIdentifierMatcher = HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN.matcher(textBlock.toString());
         boolean isAtLeast3Characters = atLeast3Matcher.reset().find();
         boolean headlineWithSlashesMatches = headlineWithSlashesMatcher.reset().matches();
@@ -148,6 +150,8 @@ public class DocuMineClassificationService {
                    && greaterOrEqualFontThanPageAverage(textBlock, page)//
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9//
                    && (tableOrFigureMatcher.reset().find() || (headlineWithSingleIdentifierMatcher.reset().find() && listIdentifiers.isEmpty())) //
+                   && tableMidSentenceMatcher.reset().results()
+                              .count() <= 1 //
                    && !isAmount//
                    && !headlineWithSlashesMatches) {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/TableOfContentsClassificationService.java

@@ -71,6 +71,9 @@ public class TableOfContentsClassificationService {
 
     private int identifyTOCItems(int start, List<TextBlockOnPage> textBlocks, ClassificationDocument document) {
 
+        if (start >= textBlocks.size()) {
+            return start;
+        }
         ClassificationPage startPage = textBlocks.get(start).page();
         List<TextBlockOnPage> initialLookAhead = textBlocks.subList(start, Math.min(start + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()));
         HashMap<NumberWord, TextBlockOnPage> numberToBlockLookup = new HashMap<>();