.gitlab-ci.yml

@@ -1,7 +1,3 @@
-variables:
-  #  SONAR_PROJECT_KEY: 'ocr-service:ocr-service-server'
-  GIT_SUBMODULE_STRATEGY: recursive
-  GIT_SUBMODULE_FORCE_HTTPS: 'true'
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'

.gitmodules

@@ -1,8 +0,0 @@
-[submodule "ocr-service-v1/ocr-service-server/src/test/resources/files/syngenta"]
-	path = ocr-service-v1/ocr-service-server/src/test/resources/files/syngenta
-	url = https://gitlab.knecon.com/fforesight/documents/syngenta.git
-	update = merge
-[submodule "ocr-service-v1/ocr-service-server/src/test/resources/files/basf"]
-	path = ocr-service-v1/ocr-service-server/src/test/resources/files/basf
-	url = https://gitlab.knecon.com/fforesight/documents/basf.git
-	update = merge

ocr-service-v1/ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/DocumentRequest.java

@@ -13,12 +13,5 @@ public class DocumentRequest {
 
     protected String dossierId;
     protected String fileId;
-    protected boolean removeWatermark;
 
-    public DocumentRequest(String dossierId, String fileId) {
-
-        this.dossierId = dossierId;
-        this.fileId = fileId;
-    }
-    
 }

ocr-service-v1/ocr-service-processor/build.gradle.kts

@@ -15,7 +15,6 @@ dependencies {
     api("com.iqser.red.commons:metric-commons:2.1.0")
     api("com.iqser.red.commons:storage-commons:2.45.0")
     api("com.knecon.fforesight:tenant-commons:0.21.0")
-    api("com.knecon.fforesight:lifecycle-commons:0.6.0")
     api("com.pdftron:PDFNet:10.5.0")
     api("org.apache.pdfbox:pdfbox:3.0.0")
     api("org.apache.pdfbox:jbig2-imageio:3.0.4")
@@ -25,7 +24,7 @@ dependencies {
     api("io.github.karols:hocr4j:0.2.0")
     api("com.amazonaws:aws-java-sdk-kms:1.12.440")
     api("com.google.guava:guava:31.1-jre")
-    api("com.iqser.red.commons:pdftron-logic-commons:2.32.0")
+    api("com.iqser.red.commons:pdftron-logic-commons:2.23.0")
-    api("com.knecon.fforesight:viewer-doc-processor:0.125.0")
+    api("com.knecon.fforesight:viewer-doc-processor:0.89.0")
     testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/initializer/PDFNetInitializer.java

@@ -1,20 +1,17 @@
 package com.knecon.fforesight.service.ocr.processor.initializer;
 
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
 import com.pdftron.pdf.PDFNet;
 import com.sun.jna.NativeLibrary;
 
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
 
-@Slf4j
 @Component
 @RequiredArgsConstructor
-public class NativeLibrariesInitializer {
+public class PDFNetInitializer {
 
     @Value("${pdftron.license:}")
     private String pdftronLicense;
@@ -25,25 +22,8 @@ public class NativeLibrariesInitializer {
     // Do not change back to application runner, if it is application runner it takes messages from the queue before PDFNet is initialized, that leads to UnsatisfiedLinkError.
     public void init() {
 
-        log.info("Initializing Native Libraries");
-        log.info("Setting pdftron license: {}", pdftronLicense);
         PDFNet.setTempPath("/tmp/pdftron");
         PDFNet.initialize(pdftronLicense);
-
-        log.info("Setting jna.library.path: {}", System.getenv("VCPKG_DYNAMIC_LIB"));
         System.setProperty("jna.library.path", System.getenv("VCPKG_DYNAMIC_LIB"));
-
-        log.info("Asserting Native Libraries loaded");
-
-        try (NativeLibrary leptonicaLib = NativeLibrary.getInstance("leptonica")) {
-            assert leptonicaLib != null;
-            log.info("Leptonica library loaded from {}", leptonicaLib.getFile().getAbsolutePath());
-        }
-
-        try (NativeLibrary tesseractLib = NativeLibrary.getInstance("tesseract")) {
-            assert tesseractLib != null;
-            log.info("Tesseract library loaded from {}", tesseractLib.getFile().getAbsolutePath());
-        }
     }
-
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/PageInformation.java

@@ -1,42 +1,12 @@
 package com.knecon.fforesight.service.ocr.processor.model;
 
-import java.awt.geom.Rectangle2D;
-
 import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
 
-public record PageInformation(Rectangle2D mediabox, int number, int rotationDegrees) {
+public record PageInformation(int height, int width, int number, int rotationDegrees) {
 
     public static PageInformation fromPDPage(int pageNum, PDPage page) {
 
-        PDRectangle mediaBox = page.getMediaBox();
+        return new PageInformation((int) page.getMediaBox().getHeight(), (int) page.getMediaBox().getWidth(), pageNum, page.getRotation());
-        return new PageInformation(new Rectangle2D.Double(mediaBox.getLowerLeftX(), mediaBox.getLowerLeftY(), mediaBox.getWidth(), mediaBox.getHeight()),
-                                   pageNum,
-                                   page.getRotation());
-    }
-
-
-    public double height() {
-
-        return mediabox.getHeight();
-    }
-
-
-    public double width() {
-
-        return mediabox.getWidth();
-    }
-
-
-    public double minX() {
-
-        return mediabox.getX();
-    }
-
-
-    public double minY() {
-
-        return mediabox.getY();
     }
 
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/QuadPoint.java

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.ocr.processor.model;
 
-import java.awt.Rectangle;
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
@@ -35,16 +34,6 @@ public record QuadPoint(Point2D a, Point2D b, Point2D c, Point2D d) {
                 new Point2D.Double(bounds.getRight(), bounds.getBottom()));
     }
 
-    public Rectangle2D getBounds2D() {
-
-        double minX = Math.min(Math.min(Math.min(a.getX(), b.getX()), c.getX()), d.getX());
-        double minY = Math.min(Math.min(Math.min(a.getY(), b.getY()), c.getY()), d.getY());
-        double maxX = Math.max(Math.max(Math.max(a.getX(), b.getX()), c.getX()), d.getX());
-        double maxY = Math.max(Math.max(Math.max(a.getY(), b.getY()), c.getY()), d.getY());
-
-        return new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY);
-    }
-
 
     public QuadPoint getTransformed(AffineTransform at) {
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/RenderedPageOcrImage.java

@@ -1,12 +1,20 @@
 package com.knecon.fforesight.service.ocr.processor.model;
 
 import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
 
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
+import net.sourceforge.lept4j.Leptonica1;
 import net.sourceforge.lept4j.Pix;
+import net.sourceforge.tess4j.ITessAPI;
 
 @Getter
 @RequiredArgsConstructor

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/OCRService.java

@@ -67,9 +67,9 @@ public class OCRService {
      */
     @Observed(name = "OCRService", contextualName = "run-ocr-on-document")
     @SneakyThrows
-    public void runOcrOnDocument(String dossierId, String fileId, boolean removeWatermark, Path tmpDir, File documentFile, File viewerDocumentFile) {
+    public void runOcrOnDocument(String dossierId, String fileId, Path tmpDir, File documentFile, File viewerDocumentFile) {
 
-        if (removeWatermark) {
+        if (settings.isRemoveWatermark()) {
             removeWatermarkIfEnabled(documentFile);
         }
         removeInvisibleElements(documentFile);

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/OcrResultWriter.java

@@ -3,23 +3,19 @@ package com.knecon.fforesight.service.ocr.processor.service;
 import java.awt.Color;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
 import java.io.File;
-import java.io.FileInputStream;
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.pdfbox.pdmodel.graphics.state.RenderingMode;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.pdftronlogic.commons.Converter;
 import com.knecon.fforesight.service.ocr.processor.model.OcrResultToWrite;
 import com.knecon.fforesight.service.ocr.processor.model.QuadPoint;
 import com.knecon.fforesight.service.ocr.processor.model.TextPositionInImage;
@@ -30,9 +26,6 @@ import com.knecon.fforesight.service.viewerdoc.model.PlacedText;
 import com.knecon.fforesight.service.viewerdoc.model.Visualizations;
 import com.knecon.fforesight.service.viewerdoc.model.VisualizationsOnPage;
 import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
-import com.pdftron.pdf.PDFDoc;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.TextExtractor;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -46,161 +39,96 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class OcrResultWriter {
 
-    public static final Color REGULAR_TEXT_COLOR = Color.BLUE;
-    public static final Color BOLD_TEXT_COLOR = Color.CYAN;
-
-    public static final Color REGULAR_TEXT_IN_IGNORE_ZONE = Color.RED;
-    public static final Color BOLD_TEXT_IN_IGNORE_ZONE = Color.RED;
-
     ViewerDocumentService viewerDocumentService;
 
 
     @SneakyThrows
     public void drawOcrResultsToPdf(File document, File viewerDocument, Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
 
-        Map<Integer, VisualizationsOnPage> ocrVisualizationsOnPages = new HashMap<>();
+        Map<Integer, VisualizationsOnPage> ocrVisualizationsOnPages = createVisualizations(imagesWithResultsPerPage);
-        Map<Integer, VisualizationsOnPage> ocrTextDebugVisualizationsOnPages = new HashMap<>();
+        Map<Integer, VisualizationsOnPage> ocrTextDebugVisualizationsOnPages = createDebugTextVisualizations(imagesWithResultsPerPage);
-        Map<Integer, VisualizationsOnPage> ocrBBoxDebugVisualizationsOnPages = new HashMap<>();
+        Map<Integer, VisualizationsOnPage> ocrBBoxDebugVisualizationsOnPages = createDebugBBoxVisualizations(imagesWithResultsPerPage);
-
-        try (var in = new FileInputStream(document); PDFDoc doc = new PDFDoc(in)) {
-
-            for (Integer pageNumber : imagesWithResultsPerPage.keySet()) {
-
-                List<Rectangle2D> textBBoxes = getTextBBoxes(doc.getPage(pageNumber));
-
-                ocrVisualizationsOnPages.put(pageNumber - 1, createVisualizations(imagesWithResultsPerPage.get(pageNumber), textBBoxes));
-                ocrTextDebugVisualizationsOnPages.put(pageNumber - 1, createDebugTextVisualizations(imagesWithResultsPerPage.get(pageNumber), textBBoxes));
-                ocrBBoxDebugVisualizationsOnPages.put(pageNumber - 1, createDebugBBoxVisualizations(imagesWithResultsPerPage.get(pageNumber)));
-            }
-        }
 
         Visualizations visualizations = new Visualizations(ContentStreams.KNECON_OCR, ocrVisualizationsOnPages, false);
 
         List<Visualizations> debugVisualizations = List.of(visualizations,
-                                                           new Visualizations(ContentStreams.KNECON_OCR_TEXT_DEBUG, ocrTextDebugVisualizationsOnPages, false),
+                new Visualizations(ContentStreams.KNECON_OCR_TEXT_DEBUG, ocrTextDebugVisualizationsOnPages, false),
-                                                           new Visualizations(ContentStreams.KNECON_OCR_BBOX_DEBUG, ocrBBoxDebugVisualizationsOnPages, false));
+                new Visualizations(ContentStreams.KNECON_OCR_BBOX_DEBUG, ocrBBoxDebugVisualizationsOnPages, false));
 
-        viewerDocumentService.addVisualizationsOnPage(document, document, List.of(visualizations));
+        viewerDocumentService.addVisualizationsOnPage(document, document, visualizations);
         viewerDocumentService.addVisualizationsOnPage(viewerDocument, viewerDocument, debugVisualizations);
     }
 
 
-    @SuppressWarnings("PMD")
+    private Map<Integer, VisualizationsOnPage> createVisualizations(Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
-    private List<Rectangle2D> getTextBBoxes(Page page) {
 
-        List<Rectangle2D> textBBoxes = new ArrayList<>();
+        return imagesWithResultsPerPage.keySet()
-        try (var textExtractor = new TextExtractor()) {
+                .stream()
-            textExtractor.begin(page);
+                .collect(Collectors.toMap(pageNumber1IdxTo0IdxMapper(), pageNumber -> createVisualizations(imagesWithResultsPerPage.get(pageNumber))));
-            try {
-
-                for (TextExtractor.Line line = textExtractor.getFirstLine(); line.isValid(); line = getNextLine(line)) {
-                    for (TextExtractor.Word word = line.getFirstWord(); word.isValid(); word = getNextWord(word)) {
-                        textBBoxes.add(Converter.toRectangle2D(word.getBBox()));
-                    }
-                }
-            } catch (Exception e) {
-                log.warn("Could not get word dimension, {}", e.getMessage());
-            }
-            return textBBoxes;
-        }
     }
 
 
-    private static TextExtractor.Word getNextWord(TextExtractor.Word word) {
+    private static Function<Integer, Integer> pageNumber1IdxTo0IdxMapper() {
-
+        // PDFBox uses a 0-based index for page numbers internally, while we use a 1-based index
-        TextExtractor.Word nextWord = word.getNextWord();
+        return p -> p - 1;
-        word.close();
-        return nextWord;
     }
 
 
-    private static TextExtractor.Line getNextLine(TextExtractor.Line line) {
+    private VisualizationsOnPage createVisualizations(List<OcrResultToWrite> ocrResultsToWrite) {
-
-        TextExtractor.Line newLine = line.getNextLine();
-        line.close();
-        return newLine;
-    }
-
-
-    private VisualizationsOnPage createVisualizations(List<OcrResultToWrite> ocrResultsToWrite, List<Rectangle2D> ignoreZones) {
-
-        List<TextPositionInImage> words = ocrResultsToWrite.stream()
-                .map(OcrResultToWrite::textPositionInImage)
-                .flatMap(Collection::stream)
-                .filter(word -> ignoreZones.stream()
-                        .noneMatch(ignoreZone -> word.getTransformedTextBBox().getBounds2D().intersects(ignoreZone)))
-                .toList();
 
+        List<TextPositionInImage> words = ocrResultsToWrite.stream().map(OcrResultToWrite::textPositionInImage).flatMap(Collection::stream).toList();
         List<PlacedText> placedTexts = words.stream()
                 .map(word -> new PlacedText(word.getText(),
-                                            null,
+                        null,
-                                            Color.BLACK,
+                        Color.BLACK,
-                                            (float) word.getFontSize(),
+                        (float) word.getFontSize(),
-                                            word.getFontMetricsFactory(),
+                        word.getFontMetricsFactory(),
-                                            Optional.of(word.getTextMatrix()),
+                        Optional.of(word.getTextMatrix()),
-                                            Optional.of(RenderingMode.NEITHER)))
+                        Optional.of(RenderingMode.NEITHER)))
                 .toList();
         return VisualizationsOnPage.builder().placedTexts(placedTexts).build();
     }
 
 
-    private VisualizationsOnPage createDebugTextVisualizations(List<OcrResultToWrite> ocrResultsToWrite, List<Rectangle2D> textBBoxes) {
+    private Map<Integer, VisualizationsOnPage> createDebugTextVisualizations(Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
 
-        List<TextPositionInImage> wordsToDraw = new ArrayList<>();
+        return imagesWithResultsPerPage.keySet()
-        List<TextPositionInImage> ignoredWords = new ArrayList<>();
+                .stream()
+                .collect(Collectors.toMap(pageNumber1IdxTo0IdxMapper(), pageNumber -> createDebugTextVisualizations(imagesWithResultsPerPage.get(pageNumber))));
+    }
 
-        for (OcrResultToWrite ocrResultToWrite : ocrResultsToWrite) {
+
-            for (TextPositionInImage textPositionInImage : ocrResultToWrite.textPositionInImage()) {
+    private VisualizationsOnPage createDebugTextVisualizations(List<OcrResultToWrite> ocrResultsToWrite) {
-                if (textBBoxes.stream()
+
-                        .anyMatch(ignoreZone -> textPositionInImage.getTransformedTextBBox().getBounds2D().intersects(ignoreZone))) {
+        List<TextPositionInImage> words = ocrResultsToWrite.stream().map(OcrResultToWrite::textPositionInImage).flatMap(Collection::stream).toList();
-                    ignoredWords.add(textPositionInImage);
+        List<PlacedText> placedTexts = words.stream()
-                } else {
-                    wordsToDraw.add(textPositionInImage);
-                }
-            }
-        }
-        Stream<PlacedText> placedTexts = wordsToDraw.stream()
                 .map(word -> new PlacedText(word.getText(),
-                                            null,
+                        null,
-                                            word.getFontStyle().equals(FontStyle.REGULAR) ? REGULAR_TEXT_COLOR : BOLD_TEXT_COLOR,
+                        word.getFontStyle().equals(FontStyle.REGULAR) ? Color.BLUE : Color.RED,
-                                            (float) word.getFontSize(),
+                        (float) word.getFontSize(),
-                                            word.getFontMetricsFactory(),
+                        word.getFontMetricsFactory(),
-                                            Optional.of(word.getTextMatrix()),
+                        Optional.of(word.getTextMatrix()),
-                                            Optional.of(RenderingMode.FILL)));
+                        Optional.of(RenderingMode.FILL)))
+                .toList();
+        return VisualizationsOnPage.builder().placedTexts(placedTexts).build();
+    }
 
-        Stream<PlacedText> placedTexts2 = ignoredWords.stream()
-                .map(word -> new PlacedText(word.getText(),
-                                            null,
-                                            word.getFontStyle().equals(FontStyle.REGULAR) ? REGULAR_TEXT_IN_IGNORE_ZONE : BOLD_TEXT_IN_IGNORE_ZONE,
-                                            (float) word.getFontSize(),
-                                            word.getFontMetricsFactory(),
-                                            Optional.of(word.getTextMatrix()),
-                                            Optional.of(RenderingMode.FILL)));
 
-        return VisualizationsOnPage.builder()
+    private Map<Integer, VisualizationsOnPage> createDebugBBoxVisualizations(Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
-                .placedTexts(Stream.of(placedTexts, placedTexts2)
+
-                                     .flatMap(Function.identity())
+        return imagesWithResultsPerPage.keySet()
-                                     .toList())
+                .stream()
-                .build();
+                .collect(Collectors.toMap(pageNumber1IdxTo0IdxMapper(), pageNumber -> createDebugBBoxVisualizations(imagesWithResultsPerPage.get(pageNumber))));
     }
 
 
     private VisualizationsOnPage createDebugBBoxVisualizations(List<OcrResultToWrite> ocrResultsToWrite) {
 
-        List<TextPositionInImage> words = ocrResultsToWrite.stream()
+        List<TextPositionInImage> words = ocrResultsToWrite.stream().map(OcrResultToWrite::textPositionInImage).flatMap(Collection::stream).toList();
-                .map(OcrResultToWrite::textPositionInImage)
-                .flatMap(Collection::stream)
-                .toList();
         List<ColoredLine> coloredLines = Stream.concat(//
-                                                       words.stream()
+                words.stream().map(TextPositionInImage::getTransformedTextBBox).map(this::quadPointAsLines),//
-                                                               .map(TextPositionInImage::getTransformedTextBBox)
+                ocrResultsToWrite.stream().map(OcrResultToWrite::imageBoundingBox).map(this::createGrid)//
-                                                               .map(this::quadPointAsLines),//
+        ).flatMap(Collection::stream).toList();
-                                                       ocrResultsToWrite.stream()
-                                                               .map(OcrResultToWrite::imageBoundingBox)
-                                                               .map(this::createGrid)//
-                )
-                .flatMap(Collection::stream)
-                .toList();
         return VisualizationsOnPage.builder().coloredLines(coloredLines).build();
     }
 
@@ -208,9 +136,9 @@ public class OcrResultWriter {
     private List<ColoredLine> quadPointAsLines(QuadPoint rect) {
 
         return List.of(new ColoredLine(new Line2D.Double(rect.a(), rect.b()), Color.ORANGE, 1),
-                       new ColoredLine(new Line2D.Double(rect.b(), rect.c()), Color.BLUE, 1),
+                new ColoredLine(new Line2D.Double(rect.b(), rect.c()), Color.BLUE, 1),
-                       new ColoredLine(new Line2D.Double(rect.c(), rect.d()), Color.GREEN, 1),
+                new ColoredLine(new Line2D.Double(rect.c(), rect.d()), Color.GREEN, 1),
-                       new ColoredLine(new Line2D.Double(rect.d(), rect.a()), Color.MAGENTA, 1));
+                new ColoredLine(new Line2D.Double(rect.d(), rect.a()), Color.MAGENTA, 1));
     }
 
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/threads/ImageProcessingThread.java

@@ -45,7 +45,7 @@ public class ImageProcessingThread extends Thread {
     final BlockingQueue<UnprocessedImage> imageInputQueue;
     final BlockingQueue<OcrImage> imageOutputQueue;
     final ITessAPI.TessBaseAPI detectionScriptHandle = initDetectionScriptHandle();
-    final L_Kernel gaussianKernel = Leptonica1.makeGaussianKernel(2, 2, 1.0f, 1);
+    final L_Kernel gaussianKernel = Leptonica1.makeGaussianKernel(2, 2, 1.2f, 1);
     final Statistics stats;
     final OcrServiceSettings settings;
     final PDDocument document;
@@ -227,7 +227,7 @@ public class ImageProcessingThread extends Thread {
         if (pix.w < 100 || pix.h < 100) {
             binarized = Leptonica1.pixThresholdToBinary(gaussian, 170);
         } else {
-            binarized = Leptonica1.pixOtsuThreshOnBackgroundNorm(gaussian, null, 50, 50, 165, 10, 100, 5, 5, 0.1f, null);
+            binarized = Leptonica1.pixOtsuThreshOnBackgroundNorm(gaussian, null, 50, 50, 165, 10, 100, 5, 5, 0.2f, null);
             if (binarized == null) { // Sometimes Otsu just fails, then we binarize directly
                 binarized = Leptonica1.pixThresholdToBinary(gaussian, 170);
             }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/settings/OcrServiceSettings.java

@@ -21,8 +21,10 @@ public class OcrServiceSettings {
     int minImageWidth = 20; // Minimum width for images to be processed
     float minRotationConfidence = 2; // Sets a lower bound for the confidence rating for rotated pages.
     boolean debug; // If true, overlays OCR images with a grid and draws word bounding boxes
+    boolean removeWatermark; // If true, watermarks will be removed
     String languages = "deu+eng"; // Defines languages loaded into Tesseract as 3-char codes, additional languages must also be installed in the docker environment
     COSName ocrMarkedContentTag = COSName.getPDFName("KNECON_OCR");
     boolean boldDetection = true; // if true, bold detection will be attempted
     double boldThreshold = 0.5; // Words are opened with a brick of average stroke width, if the ratio of remaining pixels is higher the word is determined bold.
+    boolean sendStatusUpdates = true;
 }

ocr-service-v1/ocr-service-server/build.gradle.kts

@@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
     application
     id("com.iqser.red.service.java-conventions")
-    id("org.springframework.boot") version "3.2.3"
+    id("org.springframework.boot") version "3.1.5"
     id("io.spring.dependency-management") version "1.1.3"
     id("org.sonarqube") version "4.3.0.3225"
     id("io.freefair.lombok") version "8.4"
@@ -17,14 +17,14 @@ configurations {
     }
 }
 
-val springBootStarterVersion = "3.2.3"
+val springBootStarterVersion = "3.1.5"
 
 dependencies {
     implementation(project(":ocr-service-processor"))
     implementation(project(":ocr-service-api"))
 
-    implementation("com.knecon.fforesight:tracing-commons:0.7.0")
+    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
-    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.1.1")
+    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
     implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
 
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
@@ -39,7 +39,7 @@ tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
     environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
-    environment.put("BPE_GS_LIB", "/layers/fagiani_apt/apt/usr/share/ghostscript/9.55.0/Resource/Init/") // set ghostscript lib path, version in path must match version in Aptfile
+    environment.put("BPE_GS_LIB", "/layers/fagiani_apt/apt/usr/share/ghostscript/9.26/Resource/Init/") // set ghostscript lib path
     environment.put("BPE_FONTCONFIG_PATH", "/layers/fagiani_apt/apt/etc/fonts/") // set ghostscript fontconfig path
 
     var aptfile = layout.projectDirectory.file("src/main/resources/Aptfile").toString()
@@ -53,7 +53,7 @@ tasks.named<BootBuildImage>("bootBuildImage") {
 
     buildpacks.set(
         listOf(
-            "ghcr.io/knsita/buildpacks/fagiani_apt@sha256:9771d4d27d8050aee62769490b8882fffc794745c129fb98e1f33196e2c93504",
+            "ghcr.io/fagiani/buildpacks/fagiani_apt@sha256:6471c8c70f32b749e29f65ae562ac0339fecad26aa9217628c00a6c31f197dae",
             "ghcr.io/kschuettler/knecon-vcpkg@sha256:ba5e967b124de4865ff7e8f565684f752dd6e97b302e2dcf651283f6a19b98b9",
             "ghcr.io/kschuettler/knecon-tessdata@sha256:9062f728aa0340ac963bcdd6f5e740d683823a81d3f480db894da15bff72691a",
             "urn:cnb:builder:paketo-buildpacks/java"

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/Application.java

@@ -6,27 +6,23 @@ import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.annotation.EnableAsync;
 
 import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
 import com.iqser.red.pdftronlogic.commons.WatermarkRemovalService;
-import com.knecon.fforesight.lifecyclecommons.LifecycleAutoconfiguration;
 import com.knecon.fforesight.service.ocr.processor.OcrServiceProcessorConfiguration;
 import com.knecon.fforesight.service.ocr.v1.server.queue.MessagingConfiguration;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
-import com.knecon.fforesight.tracing.OpenTelemetryConfig;
 
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
 
 @EnableAsync
-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, LifecycleAutoconfiguration.class})
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
-@Import({MessagingConfiguration.class, StorageAutoConfiguration.class, OcrServiceProcessorConfiguration.class, OpenTelemetryConfig.class})
+@Import({MessagingConfiguration.class, StorageAutoConfiguration.class, OcrServiceProcessorConfiguration.class})
-@EnableAspectJAutoProxy
 public class Application {
 
     /**

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/NoStatusUpdateOcrMessageSender.java

@@ -13,7 +13,7 @@ import lombok.experimental.FieldDefaults;
 @Service
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@ConditionalOnProperty(value = "ocrService.sendStatusUpdates", havingValue = "false")
+@ConditionalOnProperty(value = "ocr-service.sendStatusUpdates", havingValue = "false")
 public class NoStatusUpdateOcrMessageSender implements IOcrMessageSender {
 
     RabbitTemplate rabbitTemplate;

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/OcrMessageReceiver.java

@@ -10,12 +10,12 @@ import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Service;
 import org.springframework.util.FileSystemUtils;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.ocr.processor.service.FileStorageService;
-import com.knecon.fforesight.service.ocr.processor.service.IOcrMessageSender;
 import com.knecon.fforesight.service.ocr.processor.service.OCRService;
 import com.knecon.fforesight.service.ocr.processor.service.OsUtils;
 import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
@@ -27,13 +27,14 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Service
 @RequiredArgsConstructor
+@ConditionalOnProperty(value = "ocr-service.sendStatusUpdates", havingValue = "true")
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class OcrMessageReceiver {
 
     FileStorageService fileStorageService;
     ObjectMapper objectMapper;
     OCRService ocrService;
-    IOcrMessageSender ocrMessageSender;
+    OcrMessageSender ocrMessageSender;
 
 
     @RabbitHandler
@@ -61,7 +62,7 @@ public class OcrMessageReceiver {
 
             fileStorageService.downloadFiles(dossierId, fileId, documentFile, viewerDocumentFile);
 
-            ocrService.runOcrOnDocument(dossierId, fileId, ocrRequestMessage.isRemoveWatermark(), tmpDir, documentFile, viewerDocumentFile);
+            ocrService.runOcrOnDocument(dossierId, fileId, tmpDir, documentFile, viewerDocumentFile);
 
             fileStorageService.storeFiles(dossierId, fileId, documentFile, viewerDocumentFile);
 

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/OcrMessageSender.java

@@ -1,36 +1,29 @@
 package com.knecon.fforesight.service.ocr.v1.server.queue;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.ocr.processor.service.IOcrMessageSender;
 import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
 import com.knecon.fforesight.service.ocr.v1.api.model.OCRStatusUpdateResponse;
 
-import jakarta.annotation.PostConstruct;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
 
-@Slf4j
 @Service
 @RequiredArgsConstructor
-@ConditionalOnProperty(value = "ocrService.sendStatusUpdates", havingValue = "true")
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class OcrMessageSender implements IOcrMessageSender {
 
     RabbitTemplate rabbitTemplate;
 
-
     public void sendOcrFinished(String fileId, int totalImages) {
 
         rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
                 OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(totalImages).numberOfOCRedPages(totalImages).ocrFinished(true).build());
     }
 
-
     public void sendOCRStarted(String fileId) {
 
         rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
@@ -47,9 +40,9 @@ public class OcrMessageSender implements IOcrMessageSender {
     }
 
 
-    public void sendOcrResponse(String dossierId, String fileId) {
+    public void sendOcrResponse(String dossierId, String fileId){
 
-        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_RESPONSE_QUEUE, new DocumentRequest(dossierId, fileId));
+        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_RESPONSE_QUEUE, new DocumentRequest(dossierId,fileId));
     }
 
 }

ocr-service-v1/ocr-service-server/src/main/resources/Aptfile

@@ -1,5 +1,5 @@
 # you can list packages
-ghostscript=9.55.0~dfsg1-0ubuntu5.9
+ghostscript
 pkg-config
 zip
 unzip
@@ -11,7 +11,6 @@ libk5crypto3
 libkrb5support0
 libkeyutils1
 libkrb5-3
-libbrotli1
 
 # or include links to specific .deb files
 # http://ftp.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb

ocr-service-v1/ocr-service-server/src/main/resources/application.yml

@@ -12,9 +12,6 @@ project.version: 1.0-SNAPSHOT
 server:
   port: 8080
 
-lifecycle:
-  base-package: com.knecon.fforesight.service.ocr
-
 spring:
   application:
     name: ocr-service
@@ -60,6 +57,3 @@ management:
       endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}
 
 pdftron.license: ${PDFTRON_LICENSE}
-
-ocrService:
-  sendStatusUpdates: true

ocr-service-v1/ocr-service-server/src/test/java/com/knecon/fforesight/service/ocr/v1/server/AbstractTest.java

@@ -24,10 +24,10 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.knecon.fforesight.service.ocr.processor.initializer.PDFNetInitializer;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
-import com.knecon.fforesight.service.ocr.processor.initializer.NativeLibrariesInitializer;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 import com.pdftron.pdf.PDFNet;
 
@@ -36,7 +36,7 @@ import lombok.SneakyThrows;
 
 @ExtendWith({SpringExtension.class, MockitoExtension.class})
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import({AbstractTest.TestConfiguration.class, NativeLibrariesInitializer.class})
+@Import({AbstractTest.TestConfiguration.class, PDFNetInitializer.class})
 @AutoConfigureObservability
 public class AbstractTest {
 

ocr-service-v1/ocr-service-server/src/test/java/com/knecon/fforesight/service/ocr/v1/server/OcrServiceIntegrationTest.java

@@ -50,9 +50,9 @@ public class OcrServiceIntegrationTest extends AbstractTest {
     @Test
     public void testOCRMetrics() {
 
-        testOCR("files/syngenta/CustomerFiles/SinglePages/Watermark_Page1_10.SYN524464 FS (A16148C) - Absorção cutânea.pdf");
+        testOCR("files/Watermark.pdf");
-        testOCR("files/syngenta/CustomerFiles/SinglePages/Watermark_Page1_10.SYN524464 FS (A16148C) - Absorção cutânea.pdf");
+        testOCR("files/Watermark.pdf");
-        testOCR("files/syngenta/CustomerFiles/SinglePages/Watermark_Page1_10.SYN524464 FS (A16148C) - Absorção cutânea.pdf");
+        testOCR("files/Watermark.pdf");
 
         var ocrOnDocumentMeter = registry.getMeters().stream().filter(m -> m.getId().getName().equalsIgnoreCase("redactmanager_runOcrOnDocument")).findAny();
         assertThat(ocrOnDocumentMeter.isPresent()).isTrue();
@@ -81,7 +81,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
     @Test
     public void testMergeImages() {
         // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
-        String text = testOCR("files/syngenta/CustomerFiles/SinglePages/merge_images - Page241_18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
+        String text = testOCR("files/merge_images.pdf");
         assertThat(text).contains("Bodyweight change of dams with live young - group mean values",
                 "Control",
                 "mg/g day",
@@ -101,7 +101,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
     @Test
     public void testOCRWatermark() {
 
-        assertThat(testOCR("files/syngenta/CustomerFiles/SinglePages/Watermark_Page1_10.SYN524464 FS (A16148C) - Absorção cutânea.pdf")).contains("syngenta");
+        assertThat(testOCR("files/Watermark.pdf")).contains("syngenta");
     }
 
 
@@ -125,7 +125,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
         Files.copy(pdfFileResource.getFile().toPath(), documentFile, StandardCopyOption.REPLACE_EXISTING);
         Files.copy(pdfFileResource.getFile().toPath(), viewerDocumentFile, StandardCopyOption.REPLACE_EXISTING);
 
-        ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", false, tmpDir, documentFile.toFile(), viewerDocumentFile.toFile());
+        ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", tmpDir, documentFile.toFile(), viewerDocumentFile.toFile());
         System.out.println("File:" + documentFile);
 
         try (var fileStream = new FileInputStream(documentFile.toFile())) {
@@ -177,7 +177,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
         Files.copy(file.toPath(), documentFile, StandardCopyOption.REPLACE_EXISTING);
         Files.copy(file.toPath(), viewerDocumentFile, StandardCopyOption.REPLACE_EXISTING);
 
-        ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", false, tmpDir, documentFile.toFile(), viewerDocumentFile.toFile());
+        ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", tmpDir, documentFile.toFile(), viewerDocumentFile.toFile());
         System.out.println("File:" + documentFile);
         System.out.println("\n\n");
     }

ocr-service-v1/ocr-service-server/src/test/resources/application.yml

@@ -15,10 +15,3 @@ management:
     health.enabled: true
   endpoints.web.exposure.include: prometheus, health, metrics
   metrics.export.prometheus.enabled: true
-  tracing:
-    enabled: ${TRACING_ENABLED:false}
-    sampling:
-      probability: ${TRACING_PROBABILITY:1.0}
-  otlp:
-    tracing:
-      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}

ocr-service-v1/ocr-service-server/src/test/resources/files/basf

@@ -1 +0,0 @@
-Subproject commit 9dc6c2337dea32e63aef53271dba0692537c6605

ocr-service-v1/ocr-service-server/src/test/resources/files/syngenta

@@ -1 +0,0 @@
-Subproject commit 21fefb64bf27ca2b3329a6c69d90a27450b17930

publish-custom-image.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
-
-set -e
-
 dir=${PWD##*/}
-
 gradle assemble
 
 # Get the current Git branch
@@ -15,32 +11,5 @@ commit_hash=$(git rev-parse --short=5 HEAD)
 # Combine branch and commit hash
 buildName="${USER}-${branch}-${commit_hash}"
 
-gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${buildName}
+gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
-
+echo "nexus.knecon.com:5001/ff/${dir}-server:$buildName"
-newImageName="nexus.knecon.com:5001/ff/ocr-service-server:$buildName"
-
-echo "full image name:"
-echo ${newImageName}
-echo ""
-
-if [ -z "$1" ]; then
-    exit 0
-fi
-
-namespace=${1}
-deployment_name="ocr-service-v1"
-
-echo "deploying to ${namespace}"
-
-oldImageName=$(rancher kubectl -n ${namespace} get deployment ${deployment_name} -o=jsonpath='{.spec.template.spec.containers[*].image}')
-
-if [ "${newImageName}" = "${oldImageName}" ]; then
-    echo "Image tag did not change, redeploying..."
-    rancher kubectl rollout restart deployment ${deployment_name} -n ${namespace}
-else
-    echo "upgrading the image tag..."
-    rancher kubectl set image deployment/${deployment_name} ${deployment_name}=${newImageName} -n ${namespace}
-fi
-rancher kubectl rollout status deployment ${deployment_name} -n ${namespace}
-echo "Built ${deployment_name}:${buildName} and deployed to ${namespace}"
-