Merge branch 'RED-10752-main' into 'main'

RED-10752: Enabled prometheus

See merge request fforesight/layout-parser!267

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParserSettings.java

@@ -17,6 +17,4 @@ public class LayoutParserSettings {
 
     boolean debug;
     LayoutParsingType layoutParsingTypeOverride;
-    String pdftronLicense;
-    String layoutParserVersion;
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -20,6 +20,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.mapper.DocumentDataMapper;
@@ -87,29 +88,32 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@FieldDefaults(level = AccessLevel.PRIVATE)
 public class LayoutParsingPipeline {
 
-    ImageServiceResponseAdapter imageServiceResponseAdapter;
-    CvTableParsingAdapter cvTableParsingAdapter;
-    LayoutParsingStorageService layoutParsingStorageService;
-    SectionsBuilderService sectionsBuilderService;
-    SimplifiedSectionTextService simplifiedSectionTextService;
-    RulingCleaningService rulingCleaningService;
-    TableExtractionService tableExtractionService;
-    DocuMineBlockificationService docuMineBlockificationService;
-    RedactManagerBlockificationService redactManagerBlockificationService;
-    BlockificationPostprocessingService blockificationPostprocessingService;
-    DocstrumBlockificationService docstrumBlockificationService;
-    LayoutGridService layoutGridService;
-    ObservationRegistry observationRegistry;
-    VisualLayoutParsingAdapter visualLayoutParsingAdapter;
-    GraphicExtractorService graphicExtractorService;
-    OutlineExtractorService outlineExtractorService;
-    SectionTreeBuilderService sectionTreeBuilderService;
-    SectionTreeEnhancementService sectionTreeEnhancementService;
-    LayoutParserSettings settings;
-    ClassificationService classificationService;
+    final ImageServiceResponseAdapter imageServiceResponseAdapter;
+    final CvTableParsingAdapter cvTableParsingAdapter;
+    final LayoutParsingStorageService layoutParsingStorageService;
+    final SectionsBuilderService sectionsBuilderService;
+    final SimplifiedSectionTextService simplifiedSectionTextService;
+    final RulingCleaningService rulingCleaningService;
+    final TableExtractionService tableExtractionService;
+    final DocuMineBlockificationService docuMineBlockificationService;
+    final RedactManagerBlockificationService redactManagerBlockificationService;
+    final BlockificationPostprocessingService blockificationPostprocessingService;
+    final DocstrumBlockificationService docstrumBlockificationService;
+    final LayoutGridService layoutGridService;
+    final ObservationRegistry observationRegistry;
+    final VisualLayoutParsingAdapter visualLayoutParsingAdapter;
+    final GraphicExtractorService graphicExtractorService;
+    final OutlineExtractorService outlineExtractorService;
+    final SectionTreeBuilderService sectionTreeBuilderService;
+    final SectionTreeEnhancementService sectionTreeEnhancementService;
+    final LayoutParserSettings settings;
+    final ClassificationService classificationService;
+
+    @Value("${LAYOUT_PARSER_VERSION:}")
+    private String layoutParserVersion;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -133,7 +137,6 @@ public class LayoutParsingPipeline {
 
         LayoutParsingType layoutParsingType = settings.getLayoutParsingTypeOverride() == null //
                 ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride();
-        String layoutParserVersion = settings.getLayoutParserVersion();
 
         ClassificationDocument classificationDocument = parseLayout(layoutParsingType,
                                                                     originFile,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -24,7 +24,7 @@ import lombok.EqualsAndHashCode;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
 public class Line extends TextBoundingBox {
 
-    private static final double WORD_DISTANCE_MULTIPLIER = 0.18;
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.17;
 
     @EqualsAndHashCode.Include
     private final double x0;
@@ -157,6 +157,9 @@ public class Line extends TextBoundingBox {
 
     private void computeWords(List<Character> characters, double wordSpacing) {
 
+        // Imo, the width of space should be scaled with the font size, but it only depends on the median distance between horizontal neighbours.
+        // If there are large differences in fontsize on a page, this might lead to missing spaces for the smaller fonts and too many for larger fonts.
+        // I've just now changed the scaling factor. If you come across this comment with missing whitespaces again, try scaling the fontsize instead of simply changing the factor again.
         Word word = new Word();
         Character previous = null;
         for (Character current : characters) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClassificationPatterns.java

@@ -18,10 +18,14 @@ public class ClassificationPatterns {
 
 
 
-    public static final Pattern TABLE_OR_FIGURE_PATTERN = Pattern.compile(
+    public static final Pattern TABLE_OR_FIGURE_HEADLINE_PATTERN = Pattern.compile(
             "^\\s*(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
             Pattern.CASE_INSENSITIVE);
 
+    public static final Pattern TABLE_MID_SENTENCE_PATTERN = Pattern.compile(
+            "(?:table|continued\\s+table|appendix|figure)\\s+(?:[xvi]+|[a-z0-9]{1,3}(?:\\.[0-9]{1,3})*(?:-[0-9]{1,3})?)\\b",
+            Pattern.CASE_INSENSITIVE);
+
     public static final Pattern ALPHANUMERIC = Pattern.compile("[a-zA-Z0-9]");
 
     public static final Pattern NUMERIC = Pattern.compile("[0-9]+");

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/DocuMineClassificationService.java

@@ -6,7 +6,8 @@ import static com.knecon.fforesight.service.layoutparser.processor.services.clas
 import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_PATTERN_WITH_SLASHES;
 import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_2_IDENTIFER_PATTERN;
 import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN;
-import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_OR_FIGURE_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_MID_SENTENCE_PATTERN;
+import static com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationPatterns.TABLE_OR_FIGURE_HEADLINE_PATTERN;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -83,7 +84,8 @@ public class DocuMineClassificationService {
         Matcher atLeast3Matcher = AT_LEAST_3_CHARS_PATTERN.matcher(textBlock.toString());
         Matcher headlineWithSlashesMatcher = HEADLINE_PATTERN_WITH_SLASHES.matcher(textBlock.toString());
         Matcher amountMatcher = AMOUNT_PATTERN.matcher(textBlock.toString());
-        Matcher tableOrFigureMatcher = TABLE_OR_FIGURE_PATTERN.matcher(textBlock.toString());
+        Matcher tableOrFigureMatcher = TABLE_OR_FIGURE_HEADLINE_PATTERN.matcher(textBlock.toString());
+        Matcher tableMidSentenceMatcher = TABLE_MID_SENTENCE_PATTERN.matcher(textBlock.toString());
         Matcher headlineWithSingleIdentifierMatcher = HEADLINE_WITH_SINGLE_IDENTIFER_PATTERN.matcher(textBlock.toString());
         boolean isAtLeast3Characters = atLeast3Matcher.reset().find();
         boolean headlineWithSlashesMatches = headlineWithSlashesMatcher.reset().matches();
@@ -148,6 +150,8 @@ public class DocuMineClassificationService {
                    && greaterOrEqualFontThanPageAverage(textBlock, page)//
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9//
                    && (tableOrFigureMatcher.reset().find() || (headlineWithSingleIdentifierMatcher.reset().find() && listIdentifiers.isEmpty())) //
+                   && tableMidSentenceMatcher.reset().results()
+                              .count() <= 1 //
                    && !isAmount//
                    && !headlineWithSlashesMatches) {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/TableOfContentsClassificationService.java

@@ -71,6 +71,9 @@ public class TableOfContentsClassificationService {
 
     private int identifyTOCItems(int start, List<TextBlockOnPage> textBlocks, ClassificationDocument document) {
 
+        if (start >= textBlocks.size()) {
+            return start;
+        }
         ClassificationPage startPage = textBlocks.get(start).page();
         List<TextBlockOnPage> initialLookAhead = textBlocks.subList(start, Math.min(start + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()));
         HashMap<NumberWord, TextBlockOnPage> numberToBlockLookup = new HashMap<>();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/visualization/LayoutGrid.java

@@ -417,14 +417,15 @@ public class LayoutGrid extends LayoutGridLayerConfig {
     public void addVersionAndLayoutParsingType(String version, String layoutParsingType, Page page) {
 
         PageInformation pageInformation = PageInformation.fromPage(page);
-        Point2D point1 = new Point2D.Double(0, pageInformation.height() - 5);
-        Point2D point2 = new Point2D.Double(0, pageInformation.height() - 5 - FONT_SIZE * 1.5);
+        double startHeight = pageInformation.heightRot() - 5;
+        Point2D point1 = new Point2D.Double(0, startHeight);
+        Point2D point2 = new Point2D.Double(0, startHeight - FONT_SIZE * 1.5);
         AffineTransform affineTransform = CoordinateTransforms.calculatePageCoordsToInitialUserSpaceCoords(pageInformation);
-        Point2D transformed1 = affineTransform.transform(point1, point1);
-        Point2D transformed2 = affineTransform.transform(point2, point2);
+        affineTransform.transform(point1, point1);
+        affineTransform.transform(point2, point2);
         getOrCreateVisualizationsOnPage(page.getNumber(), this.versionAndType).getPlacedTexts()
-                .addAll(List.of(PlacedText.textFacingUp(version, transformed1, FONT_SIZE, Color.BLACK, FONT),
-                                PlacedText.textFacingUp(layoutParsingType, transformed2, FONT_SIZE, Color.BLACK, FONT)));
+                .addAll(List.of(PlacedText.textFacingUp(String.valueOf(version), point1, FONT_SIZE, Color.BLACK, FONT),
+                                PlacedText.textFacingUp(String.valueOf(layoutParsingType), point2, FONT_SIZE, Color.BLACK, FONT)));
     }
 
 

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -39,6 +39,7 @@ dependencies {
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
     implementation("ch.qos.logback:logback-classic")
+    api("com.iqser.red.commons:metric-commons:2.3.0")
 
     implementation("com.pdftron:PDFNet:10.11.0")
 

layoutparser-service/viewer-doc-processor/src/test/java/com/knecon/fforesight/service/viewerdoc/service/ViewerDocVersioningUtilityTest.java

@@ -44,7 +44,7 @@ class ViewerDocVersioningUtilityTest {
             doc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
         }
         assert ViewerDocVersioningUtility.isCurrentVersion(tmpFile.toFile());
-        //assert tmpFile.toFile().delete();
+        assert tmpFile.toFile().delete();
     }
 
 }