RED-8825: minor fixes

* also added overrides via env variables

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -96,6 +96,7 @@ public class LayoutParsingPipeline {
     VisualLayoutParsingAdapter visualLayoutParsingAdapter;
     ClarifyndClassificationService clarifyndClassificationService;
     GraphicExtractorService graphicExtractorService;
+    LayoutparserSettings settings;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -121,7 +122,8 @@ public class LayoutParsingPipeline {
             tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
         }
 
-        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+        ClassificationDocument classificationDocument = parseLayout(settings.getLayoutParsingTypeOverride() == null //
+                                                                            ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
                                                                     originFile,
                                                                     imageServiceResponse,
                                                                     tableServiceResponse,
@@ -130,7 +132,8 @@ public class LayoutParsingPipeline {
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
-        Document documentGraph = observeBuildDocumentGraph(layoutParsingRequest.layoutParsingType(), classificationDocument);
+        Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null //
+                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument);
 
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
 
@@ -224,7 +227,9 @@ public class LayoutParsingPipeline {
         Map<Integer, List<ClassifiedImage>> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse);
         ClassificationDocument classificationDocument = new ClassificationDocument();
 
-        classificationDocument.getVisualizations().setActive(identifier.containsKey("debug"));
+        if (settings.isDebug() || identifier.containsKey("debug")) {
+            classificationDocument.getVisualizations().setActive(true);
+        }
 
         List<ClassificationPage> classificationPages = new ArrayList<>();
 
@@ -266,7 +271,8 @@ public class LayoutParsingPipeline {
             classificationDocument.getVisualizations().addRulingVisualization(stripper.getRulings(), pageNumber);
             CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
 
-            List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontals(), cleanRulings.getVerticals(), PageInformation.fromPDPage(pageNumber, pdPage));
+            PageInformation pageInformation = PageInformation.fromPDPage(pageNumber, pdPage);
+            List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontals(), cleanRulings.getVerticals(), pageInformation);
             classificationDocument.getVisualizations().addCellVisualizations(emptyTableCells, pageNumber);
 
             TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(words, cleanRulings);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutparserSettings.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Configuration
+@ConfigurationProperties("layoutparser")
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class LayoutparserSettings {
+
+    boolean debug;
+    LayoutParsingType layoutParsingTypeOverride;
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/ColumnDetector.java

@@ -15,11 +15,12 @@ WIP, mostly working, needs to be tested a bit more
 public class ColumnDetector {
 
     public static final double MAX_VALUE_THRESHOLD = 0.5;
-    final static int bins_num = 128;
+    final static int bins_num = 512;
     final static int globalStartIdx = 0; // ignore outer parts completely, we don't expect columns there
     final static int globalEndIdx = bins_num; // i chose 7, since thirds seems a likely split for columns, therefore divided by 6 would eliminate those.
     public static final double DERIVATIVE_ZERO_THRESHOLD = 1e-10;
     public static final double MINIMUM_THRESHOLD_FOR_COLUMNS = 0.05;
+    public static final double NEAR_GLOBAL_THRESHOLD = 0.5;
     double minY;
     double maxY;
     double midY;
@@ -241,10 +242,10 @@ public class ColumnDetector {
         List<Integer> nearGlobalDvMaximaIdx = new LinkedList<>();
         List<Integer> nearGlobalDvMinimaIdx = new LinkedList<>();
         for (int i = globalStartIdx; i < globalEndIdx; i++) {
-            if (derivative[i] <= minDvValue * 0.8) {
+            if (derivative[i] <= minDvValue * NEAR_GLOBAL_THRESHOLD) {
                 nearGlobalDvMinimaIdx.add(i);
             }
-            if (derivative[i] >= maxDvValue * 0.8) {
+            if (derivative[i] >= maxDvValue * NEAR_GLOBAL_THRESHOLD) {
                 nearGlobalDvMaximaIdx.add(i);
             }
         }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -76,9 +76,9 @@ public class RedTextPosition extends BoundingBox {
         pos.setBBoxDirAdj(dirAdjPosition);
 
         AffineTransform affineTransform = getRotationMatrix(TextDirection.fromDegrees(textPosition.getDir()), textPosition.getPageWidth(), textPosition.getPageHeight());
-        Rectangle2D initialUserSpacePositionRect = affineTransform.createTransformedShape(dirAdjPosition).getBounds2D();
+        Rectangle2D bBoxInitialUserSpace = affineTransform.createTransformedShape(dirAdjPosition).getBounds2D();
 
-        pos.setBBoxInitialUserSpace(initialUserSpacePositionRect); // These are definitely correct
+        pos.setBBoxInitialUserSpace(bBoxInitialUserSpace); // These are definitely correct
 
         return pos;
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/RulingCleaningService.java

@@ -58,6 +58,7 @@ public class RulingCleaningService {
                                                                                                           .toList());
         List<Ruling> cleanedVerticalRulings = groupedOverlappingVerticalRectangles.stream()
                 .map(rectList -> getXCenteredRuling(RectangleTransformations.rectangle2DBBox(rectList)))
+                .filter(ruling -> ruling.length() > 0)
                 .toList();
 
         List<List<Rectangle2D>> groupedOverlappingHorizontalRectangles = groupOverlappingRectangles(rulings.horizontalLines.stream()
@@ -67,6 +68,7 @@ public class RulingCleaningService {
 
         List<Ruling> cleanedHorizontalRulings = groupedOverlappingHorizontalRectangles.stream()
                 .map(rectList -> getYCenteredRuling(RectangleTransformations.rectangle2DBBox(rectList)))
+                .filter(ruling -> ruling.length() > 0)
                 .collect(Collectors.toList());
 
         return new Rulings(cleanedVerticalRulings, cleanedHorizontalRulings);