Merge branch 'RED-7669' into 'master'

RED-7669: improve ocr

See merge request fforesight/ocr-service!51

ocr-service-v1/ocr-service-processor/build.gradle.kts

@@ -25,6 +25,6 @@ dependencies {
     api("com.amazonaws:aws-java-sdk-kms:1.12.440")
     api("com.google.guava:guava:31.1-jre")
     api("com.iqser.red.commons:pdftron-logic-commons:2.27.0")
-    api("com.knecon.fforesight:viewer-doc-processor:0.89.0")
+    api("com.knecon.fforesight:viewer-doc-processor:0.125.0")
     testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/initializer/NativeLibrariesInitializer.java

@@ -1,17 +1,20 @@
 package com.knecon.fforesight.service.ocr.processor.initializer;
 
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
 import com.pdftron.pdf.PDFNet;
 import com.sun.jna.NativeLibrary;
 
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import org.springframework.beans.factory.annotation.Value;
+import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
 
+@Slf4j
 @Component
 @RequiredArgsConstructor
-public class PDFNetInitializer {
+public class NativeLibrariesInitializer {
 
     @Value("${pdftron.license:}")
     private String pdftronLicense;
@@ -22,8 +25,25 @@ public class PDFNetInitializer {
     // Do not change back to application runner, if it is application runner it takes messages from the queue before PDFNet is initialized, that leads to UnsatisfiedLinkError.
     public void init() {
 
+        log.info("Initializing Native Libraries");
+        log.info("Setting pdftron license: {}", pdftronLicense);
         PDFNet.setTempPath("/tmp/pdftron");
         PDFNet.initialize(pdftronLicense);
+
+        log.info("Setting jna.library.path: {}", System.getenv("VCPKG_DYNAMIC_LIB"));
         System.setProperty("jna.library.path", System.getenv("VCPKG_DYNAMIC_LIB"));
+
+        log.info("Asserting Native Libraries loaded");
+
+        try (NativeLibrary leptonicaLib = NativeLibrary.getInstance("leptonica")) {
+            assert leptonicaLib != null;
+            log.info("Leptonica library loaded from {}", leptonicaLib.getFile().getAbsolutePath());
+        }
+
+        try (NativeLibrary tesseractLib = NativeLibrary.getInstance("tesseract")) {
+            assert tesseractLib != null;
+            log.info("Tesseract library loaded from {}", tesseractLib.getFile().getAbsolutePath());
+        }
     }
+
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/QuadPoint.java

@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.ocr.processor.model;
 
+import java.awt.Rectangle;
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
@@ -34,6 +35,16 @@ public record QuadPoint(Point2D a, Point2D b, Point2D c, Point2D d) {
                 new Point2D.Double(bounds.getRight(), bounds.getBottom()));
     }
 
+    public Rectangle2D getBounds2D() {
+
+        double minX = Math.min(Math.min(Math.min(a.getX(), b.getX()), c.getX()), d.getX());
+        double minY = Math.min(Math.min(Math.min(a.getY(), b.getY()), c.getY()), d.getY());
+        double maxX = Math.max(Math.max(Math.max(a.getX(), b.getX()), c.getX()), d.getX());
+        double maxY = Math.max(Math.max(Math.max(a.getY(), b.getY()), c.getY()), d.getY());
+
+        return new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY);
+    }
+
 
     public QuadPoint getTransformed(AffineTransform at) {
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/OcrResultWriter.java

@@ -3,19 +3,23 @@ package com.knecon.fforesight.service.ocr.processor.service;
 import java.awt.Color;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
 import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.pdfbox.pdmodel.graphics.state.RenderingMode;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.pdftronlogic.commons.Converter;
 import com.knecon.fforesight.service.ocr.processor.model.OcrResultToWrite;
 import com.knecon.fforesight.service.ocr.processor.model.QuadPoint;
 import com.knecon.fforesight.service.ocr.processor.model.TextPositionInImage;
@@ -26,6 +30,9 @@ import com.knecon.fforesight.service.viewerdoc.model.PlacedText;
 import com.knecon.fforesight.service.viewerdoc.model.Visualizations;
 import com.knecon.fforesight.service.viewerdoc.model.VisualizationsOnPage;
 import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.pdf.TextExtractor;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -39,96 +46,161 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class OcrResultWriter {
 
+    public static final Color REGULAR_TEXT_COLOR = Color.BLUE;
+    public static final Color BOLD_TEXT_COLOR = Color.CYAN;
+
+    public static final Color REGULAR_TEXT_IN_IGNORE_ZONE = Color.RED;
+    public static final Color BOLD_TEXT_IN_IGNORE_ZONE = Color.RED;
+
     ViewerDocumentService viewerDocumentService;
 
 
     @SneakyThrows
     public void drawOcrResultsToPdf(File document, File viewerDocument, Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
 
-        Map<Integer, VisualizationsOnPage> ocrVisualizationsOnPages = createVisualizations(imagesWithResultsPerPage);
+        Map<Integer, VisualizationsOnPage> ocrVisualizationsOnPages = new HashMap<>();
-        Map<Integer, VisualizationsOnPage> ocrTextDebugVisualizationsOnPages = createDebugTextVisualizations(imagesWithResultsPerPage);
+        Map<Integer, VisualizationsOnPage> ocrTextDebugVisualizationsOnPages = new HashMap<>();
-        Map<Integer, VisualizationsOnPage> ocrBBoxDebugVisualizationsOnPages = createDebugBBoxVisualizations(imagesWithResultsPerPage);
+        Map<Integer, VisualizationsOnPage> ocrBBoxDebugVisualizationsOnPages = new HashMap<>();
+
+        try (var in = new FileInputStream(document); PDFDoc doc = new PDFDoc(in)) {
+
+            for (Integer pageNumber : imagesWithResultsPerPage.keySet()) {
+
+                List<Rectangle2D> textBBoxes = getTextBBoxes(doc.getPage(pageNumber));
+
+                ocrVisualizationsOnPages.put(pageNumber - 1, createVisualizations(imagesWithResultsPerPage.get(pageNumber), textBBoxes));
+                ocrTextDebugVisualizationsOnPages.put(pageNumber - 1, createDebugTextVisualizations(imagesWithResultsPerPage.get(pageNumber), textBBoxes));
+                ocrBBoxDebugVisualizationsOnPages.put(pageNumber - 1, createDebugBBoxVisualizations(imagesWithResultsPerPage.get(pageNumber)));
+            }
+        }
 
         Visualizations visualizations = new Visualizations(ContentStreams.KNECON_OCR, ocrVisualizationsOnPages, false);
 
         List<Visualizations> debugVisualizations = List.of(visualizations,
-                new Visualizations(ContentStreams.KNECON_OCR_TEXT_DEBUG, ocrTextDebugVisualizationsOnPages, false),
+                                                           new Visualizations(ContentStreams.KNECON_OCR_TEXT_DEBUG, ocrTextDebugVisualizationsOnPages, false),
-                new Visualizations(ContentStreams.KNECON_OCR_BBOX_DEBUG, ocrBBoxDebugVisualizationsOnPages, false));
+                                                           new Visualizations(ContentStreams.KNECON_OCR_BBOX_DEBUG, ocrBBoxDebugVisualizationsOnPages, false));
 
-        viewerDocumentService.addVisualizationsOnPage(document, document, visualizations);
+        viewerDocumentService.addVisualizationsOnPage(document, document, List.of(visualizations));
         viewerDocumentService.addVisualizationsOnPage(viewerDocument, viewerDocument, debugVisualizations);
     }
 
 
-    private Map<Integer, VisualizationsOnPage> createVisualizations(Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
+    @SuppressWarnings("PMD")
+    private List<Rectangle2D> getTextBBoxes(Page page) {
 
-        return imagesWithResultsPerPage.keySet()
+        List<Rectangle2D> textBBoxes = new ArrayList<>();
-                .stream()
+        try (var textExtractor = new TextExtractor()) {
-                .collect(Collectors.toMap(pageNumber1IdxTo0IdxMapper(), pageNumber -> createVisualizations(imagesWithResultsPerPage.get(pageNumber))));
+            textExtractor.begin(page);
+            try {
+
+                for (TextExtractor.Line line = textExtractor.getFirstLine(); line.isValid(); line = getNextLine(line)) {
+                    for (TextExtractor.Word word = line.getFirstWord(); word.isValid(); word = getNextWord(word)) {
+                        textBBoxes.add(Converter.toRectangle2D(word.getBBox()));
+                    }
+                }
+            } catch (Exception e) {
+                log.warn("Could not get word dimension, {}", e.getMessage());
+            }
+            return textBBoxes;
+        }
     }
 
 
-    private static Function<Integer, Integer> pageNumber1IdxTo0IdxMapper() {
+    private static TextExtractor.Word getNextWord(TextExtractor.Word word) {
-        // PDFBox uses a 0-based index for page numbers internally, while we use a 1-based index
+
-        return p -> p - 1;
+        TextExtractor.Word nextWord = word.getNextWord();
+        word.close();
+        return nextWord;
     }
 
 
-    private VisualizationsOnPage createVisualizations(List<OcrResultToWrite> ocrResultsToWrite) {
+    private static TextExtractor.Line getNextLine(TextExtractor.Line line) {
+
+        TextExtractor.Line newLine = line.getNextLine();
+        line.close();
+        return newLine;
+    }
+
+
+    private VisualizationsOnPage createVisualizations(List<OcrResultToWrite> ocrResultsToWrite, List<Rectangle2D> ignoreZones) {
+
+        List<TextPositionInImage> words = ocrResultsToWrite.stream()
+                .map(OcrResultToWrite::textPositionInImage)
+                .flatMap(Collection::stream)
+                .filter(word -> ignoreZones.stream()
+                        .noneMatch(ignoreZone -> word.getTransformedTextBBox().getBounds2D().intersects(ignoreZone)))
+                .toList();
 
-        List<TextPositionInImage> words = ocrResultsToWrite.stream().map(OcrResultToWrite::textPositionInImage).flatMap(Collection::stream).toList();
         List<PlacedText> placedTexts = words.stream()
                 .map(word -> new PlacedText(word.getText(),
-                        null,
+                                            null,
-                        Color.BLACK,
+                                            Color.BLACK,
-                        (float) word.getFontSize(),
+                                            (float) word.getFontSize(),
-                        word.getFontMetricsFactory(),
+                                            word.getFontMetricsFactory(),
-                        Optional.of(word.getTextMatrix()),
+                                            Optional.of(word.getTextMatrix()),
-                        Optional.of(RenderingMode.NEITHER)))
+                                            Optional.of(RenderingMode.NEITHER)))
                 .toList();
         return VisualizationsOnPage.builder().placedTexts(placedTexts).build();
     }
 
 
-    private Map<Integer, VisualizationsOnPage> createDebugTextVisualizations(Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
+    private VisualizationsOnPage createDebugTextVisualizations(List<OcrResultToWrite> ocrResultsToWrite, List<Rectangle2D> textBBoxes) {
 
-        return imagesWithResultsPerPage.keySet()
+        List<TextPositionInImage> wordsToDraw = new ArrayList<>();
-                .stream()
+        List<TextPositionInImage> ignoredWords = new ArrayList<>();
-                .collect(Collectors.toMap(pageNumber1IdxTo0IdxMapper(), pageNumber -> createDebugTextVisualizations(imagesWithResultsPerPage.get(pageNumber))));
-    }
 
-
+        for (OcrResultToWrite ocrResultToWrite : ocrResultsToWrite) {
-    private VisualizationsOnPage createDebugTextVisualizations(List<OcrResultToWrite> ocrResultsToWrite) {
+            for (TextPositionInImage textPositionInImage : ocrResultToWrite.textPositionInImage()) {
-
+                if (textBBoxes.stream()
-        List<TextPositionInImage> words = ocrResultsToWrite.stream().map(OcrResultToWrite::textPositionInImage).flatMap(Collection::stream).toList();
+                        .anyMatch(ignoreZone -> textPositionInImage.getTransformedTextBBox().getBounds2D().intersects(ignoreZone))) {
-        List<PlacedText> placedTexts = words.stream()
+                    ignoredWords.add(textPositionInImage);
+                } else {
+                    wordsToDraw.add(textPositionInImage);
+                }
+            }
+        }
+        Stream<PlacedText> placedTexts = wordsToDraw.stream()
                 .map(word -> new PlacedText(word.getText(),
-                        null,
+                                            null,
-                        word.getFontStyle().equals(FontStyle.REGULAR) ? Color.BLUE : Color.RED,
+                                            word.getFontStyle().equals(FontStyle.REGULAR) ? REGULAR_TEXT_COLOR : BOLD_TEXT_COLOR,
-                        (float) word.getFontSize(),
+                                            (float) word.getFontSize(),
-                        word.getFontMetricsFactory(),
+                                            word.getFontMetricsFactory(),
-                        Optional.of(word.getTextMatrix()),
+                                            Optional.of(word.getTextMatrix()),
-                        Optional.of(RenderingMode.FILL)))
+                                            Optional.of(RenderingMode.FILL)));
-                .toList();
-        return VisualizationsOnPage.builder().placedTexts(placedTexts).build();
-    }
 
+        Stream<PlacedText> placedTexts2 = ignoredWords.stream()
+                .map(word -> new PlacedText(word.getText(),
+                                            null,
+                                            word.getFontStyle().equals(FontStyle.REGULAR) ? REGULAR_TEXT_IN_IGNORE_ZONE : BOLD_TEXT_IN_IGNORE_ZONE,
+                                            (float) word.getFontSize(),
+                                            word.getFontMetricsFactory(),
+                                            Optional.of(word.getTextMatrix()),
+                                            Optional.of(RenderingMode.FILL)));
 
-    private Map<Integer, VisualizationsOnPage> createDebugBBoxVisualizations(Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
+        return VisualizationsOnPage.builder()
-
+                .placedTexts(Stream.of(placedTexts, placedTexts2)
-        return imagesWithResultsPerPage.keySet()
+                                     .flatMap(Function.identity())
-                .stream()
+                                     .toList())
-                .collect(Collectors.toMap(pageNumber1IdxTo0IdxMapper(), pageNumber -> createDebugBBoxVisualizations(imagesWithResultsPerPage.get(pageNumber))));
+                .build();
     }
 
 
     private VisualizationsOnPage createDebugBBoxVisualizations(List<OcrResultToWrite> ocrResultsToWrite) {
 
-        List<TextPositionInImage> words = ocrResultsToWrite.stream().map(OcrResultToWrite::textPositionInImage).flatMap(Collection::stream).toList();
+        List<TextPositionInImage> words = ocrResultsToWrite.stream()
+                .map(OcrResultToWrite::textPositionInImage)
+                .flatMap(Collection::stream)
+                .toList();
         List<ColoredLine> coloredLines = Stream.concat(//
-                words.stream().map(TextPositionInImage::getTransformedTextBBox).map(this::quadPointAsLines),//
+                                                       words.stream()
-                ocrResultsToWrite.stream().map(OcrResultToWrite::imageBoundingBox).map(this::createGrid)//
+                                                               .map(TextPositionInImage::getTransformedTextBBox)
-        ).flatMap(Collection::stream).toList();
+                                                               .map(this::quadPointAsLines),//
+                                                       ocrResultsToWrite.stream()
+                                                               .map(OcrResultToWrite::imageBoundingBox)
+                                                               .map(this::createGrid)//
+                )
+                .flatMap(Collection::stream)
+                .toList();
         return VisualizationsOnPage.builder().coloredLines(coloredLines).build();
     }
 
@@ -136,9 +208,9 @@ public class OcrResultWriter {
     private List<ColoredLine> quadPointAsLines(QuadPoint rect) {
 
         return List.of(new ColoredLine(new Line2D.Double(rect.a(), rect.b()), Color.ORANGE, 1),
-                new ColoredLine(new Line2D.Double(rect.b(), rect.c()), Color.BLUE, 1),
+                       new ColoredLine(new Line2D.Double(rect.b(), rect.c()), Color.BLUE, 1),
-                new ColoredLine(new Line2D.Double(rect.c(), rect.d()), Color.GREEN, 1),
+                       new ColoredLine(new Line2D.Double(rect.c(), rect.d()), Color.GREEN, 1),
-                new ColoredLine(new Line2D.Double(rect.d(), rect.a()), Color.MAGENTA, 1));
+                       new ColoredLine(new Line2D.Double(rect.d(), rect.a()), Color.MAGENTA, 1));
     }
 
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/threads/ImageProcessingThread.java

@@ -227,7 +227,7 @@ public class ImageProcessingThread extends Thread {
         if (pix.w < 100 || pix.h < 100) {
             binarized = Leptonica1.pixThresholdToBinary(gaussian, 170);
         } else {
-            binarized = Leptonica1.pixOtsuThreshOnBackgroundNorm(gaussian, null, 50, 50, 165, 10, 100, 5, 5, 0.2f, null);
+            binarized = Leptonica1.pixOtsuThreshOnBackgroundNorm(gaussian, null, 50, 50, 165, 10, 100, 5, 5, 0.1f, null);
             if (binarized == null) { // Sometimes Otsu just fails, then we binarize directly
                 binarized = Leptonica1.pixThresholdToBinary(gaussian, 170);
             }

ocr-service-v1/ocr-service-server/src/test/java/com/knecon/fforesight/service/ocr/v1/server/AbstractTest.java

@@ -24,10 +24,10 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.knecon.fforesight.service.ocr.processor.initializer.PDFNetInitializer;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.service.ocr.processor.initializer.NativeLibrariesInitializer;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 import com.pdftron.pdf.PDFNet;
 
@@ -36,7 +36,7 @@ import lombok.SneakyThrows;
 
 @ExtendWith({SpringExtension.class, MockitoExtension.class})
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import({AbstractTest.TestConfiguration.class, PDFNetInitializer.class})
+@Import({AbstractTest.TestConfiguration.class, NativeLibrariesInitializer.class})
 @AutoConfigureObservability
 public class AbstractTest {