hotfix: Extend Tesseract instead of Tesseract1

README.md

@@ -74,14 +74,12 @@ String languages = "deu+eng"; // Defines languages loaded into Tesseract as 3-ch
 ```
 ## Integration
 
-The OCR-service communicates via RabbitMQ and uses the queues `ocr_request_queue`, `ocr_response_queue`,
-`ocr_dead_letter_queue`, and `ocr_status_update_response_queue`. 
+The OCR-service communicates via RabbitMQ and uses the queues `ocrQueue`, `ocrDLQ`, and `ocr_status_update_response_queue`. 
 
-### ocr_request_queue
+### ocrQueue
 This queue is used to start the OCR process, a DocumentRequest must be passed as a message. The service will then download the PDF from the provided cloud storage.
-### ocr_response_queue
-This queue is also used to signal the end of processing.
-### ocr_dead_letter_queue
-This queue is used to signal an error has occurred during processing.
 ### ocr_status_update_response_queue
 This queue is used by the OCR service to give updates about the progress of the ongoing OCR on a image per image basis. The total amount may change, when less images are found than initially assumed.
+This queue is also used to signal the end of processing.
+### ocrDLQ
+This queue is used to signal an error has occurred during processing.

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -12,10 +12,6 @@ group = "com.knecon.fforesight.service"
 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17
 
-pmd {
-    isConsoleOutput = true
-}
-
 tasks.pmdMain {
     pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
 }

config/pmd/pmd.xml

@@ -9,12 +9,12 @@
     </description>
 
     <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="NullAssignment"/>
-        <exclude name="AssignmentInOperand"/>
-        <exclude name="BeanMembersShouldSerialize"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
     </rule>
 
-</ruleset>
+</ruleset>

config/pmd/test_pmd.xml

@@ -10,13 +10,14 @@
 
 
     <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="NullAssignment"/>
-        <exclude name="AssignmentInOperand"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="AvoidFieldNameMatchingTypeName"/>
         <exclude name="TestClassWithoutTestCases"/>
-        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

ocr-service-v1/ocr-service-api/build.gradle.kts

@@ -1,7 +1,7 @@
 plugins {
     `maven-publish`
     id("com.iqser.red.service.java-conventions")
-    id("io.freefair.lombok") version "8.4"
+    id("io.freefair.lombok") version "8.2.2"
 }
 
 publishing {

ocr-service-v1/ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/DocumentRequest.java

@@ -13,12 +13,5 @@ public class DocumentRequest {
 
     protected String dossierId;
     protected String fileId;
-    protected boolean removeWatermark;
 
-    public DocumentRequest(String dossierId, String fileId) {
-
-        this.dossierId = dossierId;
-        this.fileId = fileId;
-    }
-    
 }

ocr-service-v1/ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/OCRStatusUpdateResponse.java

@@ -15,6 +15,5 @@ public class OCRStatusUpdateResponse {
     private int numberOfPagesToOCR;
     private int numberOfOCRedPages;
     private boolean ocrFinished;
-    private boolean ocrStarted;
 
 }

ocr-service-v1/ocr-service-processor/build.gradle.kts

@@ -1,6 +1,6 @@
 plugins {
     id("com.iqser.red.service.java-conventions")
-    id("io.freefair.lombok") version "8.4"
+    id("io.freefair.lombok") version "8.2.2"
 }
 
 configurations {
@@ -14,7 +14,7 @@ dependencies {
     api("net.sourceforge.tess4j:tess4j:5.8.0")
     api("com.iqser.red.commons:metric-commons:2.1.0")
     api("com.iqser.red.commons:storage-commons:2.45.0")
-    api("com.knecon.fforesight:tenant-commons:0.21.0")
+    api("com.knecon.fforesight:tenant-commons:0.19.0")
     api("com.pdftron:PDFNet:10.5.0")
     api("org.apache.pdfbox:pdfbox:3.0.0")
     api("org.apache.pdfbox:jbig2-imageio:3.0.4")
@@ -24,7 +24,6 @@ dependencies {
     api("io.github.karols:hocr4j:0.2.0")
     api("com.amazonaws:aws-java-sdk-kms:1.12.440")
     api("com.google.guava:guava:31.1-jre")
-    api("com.iqser.red.commons:pdftron-logic-commons:2.27.0")
-    api("com.knecon.fforesight:viewer-doc-processor:0.124.0")
+    api("com.iqser.red.commons:pdftron-logic-commons:2.20.0")
     testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/OcrServiceProcessorConfiguration.java

@@ -1,26 +1,14 @@
 package com.knecon.fforesight.service.ocr.processor;
 
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 
 import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
-import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
-
-import io.micrometer.observation.ObservationRegistry;
 
 @Configuration
 @ComponentScan
 @EnableConfigurationProperties(OcrServiceSettings.class)
 public class OcrServiceProcessorConfiguration {
 
-    @Bean
-    @Autowired
-    public ViewerDocumentService viewerDocumentService(ObservationRegistry registry) {
-
-        return new ViewerDocumentService(registry);
-    }
-
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/initializer/PDFNetInitializer.java

@@ -6,15 +6,12 @@ import com.sun.jna.NativeLibrary;
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
-@Slf4j
 @Component
 @RequiredArgsConstructor
-public class NativeLibrariesInitializer {
+public class PDFNetInitializer {
 
     @Value("${pdftron.license:}")
     private String pdftronLicense;
@@ -25,21 +22,8 @@ public class NativeLibrariesInitializer {
     // Do not change back to application runner, if it is application runner it takes messages from the queue before PDFNet is initialized, that leads to UnsatisfiedLinkError.
     public void init() {
 
-        log.info("Initializing Native Libraries");
-        log.info("Setting pdftron license: {}", pdftronLicense);
         PDFNet.setTempPath("/tmp/pdftron");
         PDFNet.initialize(pdftronLicense);
-
-        log.info("Setting jna.library.path: {}", System.getenv("VCPKG_DYNAMIC_LIB"));
         System.setProperty("jna.library.path", System.getenv("VCPKG_DYNAMIC_LIB"));
-
-        log.info("Asserting Native Libraries loaded");
-        NativeLibrary leptonicaLib = NativeLibrary.getInstance("leptonica");
-        assert leptonicaLib != null;
-        log.info("Leptonica library loaded from {}", leptonicaLib.getFile().getAbsolutePath());
-        NativeLibrary tesseractLib = NativeLibrary.getInstance("tesseract");
-        assert tesseractLib != null;
-        log.info("Tesseract library loaded from {}", leptonicaLib.getFile().getAbsolutePath());
     }
-
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/OcrImage.java

@@ -93,7 +93,7 @@ public interface OcrImage {
         if (getWidth() < 200 || getHeight() < 200) {
             return ITessAPI.TessPageSegMode.PSM_SINGLE_BLOCK;
         }
-        return ITessAPI.TessPageSegMode.PSM_SPARSE_TEXT;
+        return ITessAPI.TessPageSegMode.PSM_AUTO;
     } // TODO: evaluate if PSM can be dynamically chosen to increase performance
 
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/OcrResultToWrite.java

@@ -25,18 +25,11 @@ public record OcrResultToWrite(List<TextPositionInImage> textPositionInImage, Qu
                 .collect(Collectors.toMap(Map.Entry::getKey,
                         entry -> entry.getValue()
                                 .stream()
-                                .map(ocrResult -> new OcrResultToWrite(toTextPositionInImage(ocrResult, fontMetricsFactory), ocrResult.image().getImageCoordinatesInInitialUserSpace()))
+                                .map(ocrResult -> new OcrResultToWrite(ocrResult.getAllWords()
+                                        .stream()
+                                        .filter(word -> !word.isBlank())
+                                        .map(word -> new TextPositionInImage(word, ocrResult.image().getImageCTM(), fontMetricsFactory, FontStyle.REGULAR))
+                                        .toList(), ocrResult.image().getImageCoordinatesInInitialUserSpace()))
                                 .toList()));
     }
-
-
-    private static List<TextPositionInImage> toTextPositionInImage(OcrResult ocrResult, FontMetricsFactory fontMetricsFactory) {
-
-        return ocrResult.getAllWords()
-                .stream()
-                .filter(word -> !word.isBlank())
-                .map(word -> new TextPositionInImage(word, ocrResult.image().getImageCTM(), fontMetricsFactory, FontStyle.REGULAR))
-                .toList();
-    }
-
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/PageInformation.java

@@ -1,42 +1,12 @@
 package com.knecon.fforesight.service.ocr.processor.model;
 
-import java.awt.geom.Rectangle2D;
-
 import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
 
-public record PageInformation(Rectangle2D mediabox, int number, int rotationDegrees) {
+public record PageInformation(int height, int width, int number, int rotationDegrees) {
 
     public static PageInformation fromPDPage(int pageNum, PDPage page) {
 
-        PDRectangle mediaBox = page.getMediaBox();
-        return new PageInformation(new Rectangle2D.Double(mediaBox.getLowerLeftX(), mediaBox.getLowerLeftY(), mediaBox.getWidth(), mediaBox.getHeight()),
-                                   pageNum,
-                                   page.getRotation());
-    }
-
-
-    public double height() {
-
-        return mediabox.getHeight();
-    }
-
-
-    public double width() {
-
-        return mediabox.getWidth();
-    }
-
-
-    public double minX() {
-
-        return mediabox.getX();
-    }
-
-
-    public double minY() {
-
-        return mediabox.getY();
+        return new PageInformation((int) page.getMediaBox().getHeight(), (int) page.getMediaBox().getWidth(), pageNum, page.getRotation());
     }
 
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/QuadPoint.java

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.ocr.processor.model;
 
-import java.awt.Rectangle;
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
@@ -35,16 +34,6 @@ public record QuadPoint(Point2D a, Point2D b, Point2D c, Point2D d) {
                 new Point2D.Double(bounds.getRight(), bounds.getBottom()));
     }
 
-    public Rectangle2D getBounds2D() {
-
-        double minX = Math.min(Math.min(Math.min(a.getX(), b.getX()), c.getX()), d.getX());
-        double minY = Math.min(Math.min(Math.min(a.getY(), b.getY()), c.getY()), d.getY());
-        double maxX = Math.max(Math.max(Math.max(a.getX(), b.getX()), c.getX()), d.getX());
-        double maxY = Math.max(Math.max(Math.max(a.getY(), b.getY()), c.getY()), d.getY());
-
-        return new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY);
-    }
-
 
     public QuadPoint getTransformed(AffineTransform at) {
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/RenderedPageOcrImage.java

@@ -1,12 +1,20 @@
 package com.knecon.fforesight.service.ocr.processor.model;
 
 import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
 
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
+import net.sourceforge.lept4j.Leptonica1;
 import net.sourceforge.lept4j.Pix;
+import net.sourceforge.tess4j.ITessAPI;
 
 @Getter
 @RequiredArgsConstructor
@@ -24,7 +32,7 @@ public class RenderedPageOcrImage implements OcrImage {
     public AffineTransform getImageCTM() {
 
         double scalingFactor = calculateScalingFactor();
-        AffineTransform imageToCropBoxScaling = new AffineTransform(scalingFactor, 0, 0, scalingFactor, -pageInformation.minX(), -pageInformation.minY());
+        AffineTransform imageToCropBoxScaling = new AffineTransform(scalingFactor, 0, 0, scalingFactor, 0, 0);
 
         AffineTransform mirrorMatrix = new AffineTransform(1, 0, 0, -1, 0, pageInformation.height());
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/FileStorageService.java

@@ -1,11 +1,13 @@
 package com.knecon.fforesight.service.ocr.processor.service;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.InputStream;
 import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
+import org.apache.commons.io.IOUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@@ -29,38 +31,47 @@ public class FileStorageService {
         return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
     }
 
+
+    @SneakyThrows
+    public byte[] getOriginalFile(String dossierId, String fileId) {
+
+        try (InputStream inputStream = getInputStream(getStorageId(dossierId, fileId, FileType.ORIGIN))) {
+            return IOUtils.toByteArray(inputStream);
+        }
+    }
+
+
+    @SneakyThrows
+    public InputStream getOriginalFileAsStream(String dossierId, String fileId) {
+
+        return getInputStream(getStorageId(dossierId, fileId, FileType.ORIGIN));
+    }
+
+
+    public void storeOriginalFile(String dossierId, String fileId, InputStream stream) {
+
+        storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
+    }
+
+
     public boolean untouchedFileExists(String dossierId, String fileId) {
 
         return storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED));
     }
 
-    @SneakyThrows
-    public void storeFiles(String dossierId, String fileId, File documentFile, File viewerDocumentFile) {
 
-        try (var in = new FileInputStream(documentFile)) {
-            storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN), in);
-        }
-        try (var in = new FileInputStream(viewerDocumentFile)) {
-            storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.VIEWER_DOCUMENT), in);
-        }
+    public void storeUntouchedFile(String dossierId, String fileId, byte[] data) {
+
+        storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
     }
 
 
     @SneakyThrows
-    public void downloadFiles(String dossierId, String fileId, File documentFile, File viewerDocumentFile) {
+    private InputStream getInputStream(String storageId) {
 
-        storageService.downloadTo(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN), documentFile);
-        if (storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.VIEWER_DOCUMENT))) {
-            storageService.downloadTo(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.VIEWER_DOCUMENT), viewerDocumentFile);
-        } else {
-            Files.copy(documentFile.toPath(), viewerDocumentFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
-        }
-
-        if (!untouchedFileExists(dossierId, fileId)) {
-            try (var in = new FileInputStream(documentFile)) {
-                storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED), in);
-            }
-        }
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return Files.newInputStream(Paths.get(tempFile.getPath()), StandardOpenOption.DELETE_ON_CLOSE);
     }
 
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/IOcrMessageSender.java

@@ -7,10 +7,7 @@ public interface IOcrMessageSender {
 
     void sendUpdate(String fileId, int finishedImages, int totalImages);
 
-    void sendOCRStarted(String fileId);
 
     void sendOcrFinished(String fileId, int totalImages);
 
-    void sendOcrResponse(String dossierId, String fileId);
-
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/OCRService.java

@@ -1,12 +1,12 @@
 package com.knecon.fforesight.service.ocr.processor.service;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.nio.file.Files;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -27,10 +27,7 @@ import com.knecon.fforesight.service.ocr.processor.model.OcrResult;
 import com.knecon.fforesight.service.ocr.processor.service.scriptdetection.FontStyleDetector;
 import com.knecon.fforesight.service.ocr.processor.service.threads.OCRThread;
 import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
-import com.pdftron.pdf.PDFDoc;
 
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -51,7 +48,6 @@ public class OCRService {
     OcrResultWriter ocrResultWriter;
     GhostScriptService ghostScriptService;
     FontStyleDetector boldDetector;
-    ObservationRegistry registry;
 
 
     /**
@@ -59,66 +55,54 @@ public class OCRService {
      * looking for stitchedImages (if so converting the current page to an image with ghostscript and work on this instead),
      * perform tesseract-ocr on these images (via threads) and write the generated ocr-text as invisible elements.
      *
-     * @param dossierId          Id of dossier
-     * @param fileId             Id of file
-     * @param tmpDir             working directory for all files
-     * @param documentFile       the file to perform ocr on, results are written invisibly
-     * @param viewerDocumentFile debugging file, results are written visibly in an optional content group
+     * @param dossierId Id of dossier
+     * @param fileId    Id of file
+     * @param out       OutputStream where to write to
      */
-    @Observed(name = "OCRService", contextualName = "run-ocr-on-document")
     @SneakyThrows
-    public void runOcrOnDocument(String dossierId, String fileId, boolean removeWatermark, Path tmpDir, File documentFile, File viewerDocumentFile) {
+    public void runOcrOnDocument(String dossierId, String fileId, OutputStream out) {
 
-        if (removeWatermark) {
-            removeWatermarkIfEnabled(documentFile);
+        try (InputStream fileStream = removeWatermarkIfEnabled(dossierId, fileId); ByteArrayOutputStream transferOutputStream = new ByteArrayOutputStream()) {
+
+            invisibleElementRemovalService.removeInvisibleElements(fileStream, transferOutputStream, false);
+
+            try (InputStream transferInputStream = new ByteArrayInputStream(transferOutputStream.toByteArray())) {
+                log.info("Starting OCR for file {}", fileId);
+                long ocrStart = System.currentTimeMillis();
+                Statistics stats = runOcr(transferInputStream, out, fileId, dossierId);
+                long ocrEnd = System.currentTimeMillis();
+                log.info("ocr successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, String.format("%.1f", (ocrEnd - ocrStart) / 1000.0));
+                log.info("Runtime breakdown: {}", stats);
+            }
         }
-        removeInvisibleElements(documentFile);
+    }
 
-        log.info("Starting OCR for file {}", fileId);
-        long ocrStart = System.currentTimeMillis();
-        Statistics stats = runOcr(tmpDir, documentFile, viewerDocumentFile, fileId, dossierId);
-        long ocrEnd = System.currentTimeMillis();
-        log.info("ocr successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, String.format("%.1f", (ocrEnd - ocrStart) / 1000.0));
-        log.info("Runtime breakdown: {}", stats);
 
+    private InputStream removeWatermarkIfEnabled(String dossierId, String fileId) throws IOException {
+
+        if (settings.isRemoveWatermark()) {
+            try (var in = fileStorageService.getOriginalFileAsStream(dossierId, fileId); var transferOutputStream = new ByteArrayOutputStream()) {
+                watermarkRemovalService.removeWatermarks(in, transferOutputStream);
+                return new ByteArrayInputStream(transferOutputStream.toByteArray());
+            }
+        }
+        return fileStorageService.getOriginalFileAsStream(dossierId, fileId);
     }
 
 
     @SneakyThrows
-    private void removeInvisibleElements(File originFile) {
-
-        Path tmpFile = Files.createTempFile("invisibleElements", ".pdf");
-        try (var in = new FileInputStream(originFile); var out = new FileOutputStream(tmpFile.toFile())) {
-            invisibleElementRemovalService.removeInvisibleElements(in, out, false, false);
-        }
-        Files.copy(tmpFile, originFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
-        assert tmpFile.toFile().delete();
-    }
-
-
-    @SneakyThrows
-    private void removeWatermarkIfEnabled(File originFile) {
-
-        Path tmpFile = Files.createTempFile("removeWatermarks", ".pdf");
-        try (var in = new FileInputStream(originFile); var out = new FileOutputStream(tmpFile.toFile())) {
-            watermarkRemovalService.removeWatermarks(in, out);
-        }
-        Files.copy(tmpFile, originFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
-        assert tmpFile.toFile().delete();
-    }
-
-
-    @SneakyThrows
-    public Statistics runOcr(Path tmpDir, File documentFile, File viewerDocumentFile, String fileId, String dossierId) {
+    public Statistics runOcr(InputStream in, OutputStream out, String fileId, String dossierId) {
 
         long timestamp;
-
+        Path tmpDir = Path.of(OsUtils.getTemporaryDirectory()).resolve(dossierId + "-" + fileId);
         Path tmpImageDir = tmpDir.resolve("images");
         Path tesseractOutputDir = tmpDir.resolve("tesseract_output");
 
         tesseractOutputDir.toFile().mkdirs();
         tmpImageDir.toFile().mkdirs();
 
+        File documentFile = OsUtils.writeFileToTmpFolder(in, tmpDir);
+
         Statistics stats;
         try (PDDocument document = Loader.loadPDF(documentFile)) {
             OcrProgressLogger logger = new OcrProgressLogger(document.getNumberOfPages(), ocrMessageSender, fileId);
@@ -162,11 +146,12 @@ public class OCRService {
             stats.increaseFontStyleDetectionDuration(System.currentTimeMillis() - timestamp);
 
             timestamp = System.currentTimeMillis();
-            ocrResultWriter.drawOcrResultsToPdf(documentFile, viewerDocumentFile, imageWithTextPositionsPerPage);
-
+            var dictionariesToUpdate = ocrResultWriter.drawOcrResultsToPdf(document, imageWithTextPositionsPerPage);
             log.info("Saving document");
+            document.saveIncremental(out, dictionariesToUpdate);
             stats.increaseWritingTextDuration(System.currentTimeMillis() - timestamp);
 
+            FileSystemUtils.deleteRecursively(tmpDir);
             logger.sendFinished();
             return stats;
         }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/OcrResultWriter.java

@@ -1,39 +1,29 @@
 package com.knecon.fforesight.service.ocr.processor.service;
 
 import java.awt.Color;
-import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
-import java.io.File;
-import java.io.FileInputStream;
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import java.util.Set;
 
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.PDResources;
+import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentGroup;
+import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentProperties;
 import org.apache.pdfbox.pdmodel.graphics.state.RenderingMode;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.pdftronlogic.commons.Converter;
 import com.knecon.fforesight.service.ocr.processor.model.OcrResultToWrite;
 import com.knecon.fforesight.service.ocr.processor.model.QuadPoint;
 import com.knecon.fforesight.service.ocr.processor.model.TextPositionInImage;
-import com.knecon.fforesight.service.ocr.processor.service.fonts.FontStyle;
-import com.knecon.fforesight.service.viewerdoc.ContentStreams;
-import com.knecon.fforesight.service.viewerdoc.model.ColoredLine;
-import com.knecon.fforesight.service.viewerdoc.model.PlacedText;
-import com.knecon.fforesight.service.viewerdoc.model.Visualizations;
-import com.knecon.fforesight.service.viewerdoc.model.VisualizationsOnPage;
-import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
-import com.pdftron.pdf.PDFDoc;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.TextExtractor;
+import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -47,170 +37,180 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class OcrResultWriter {
 
-    public static final Color REGULAR_TEXT_COLOR = Color.BLUE;
-    public static final Color BOLD_TEXT_COLOR = Color.RED;
-
-    public static final Color REGULAR_TEXT_IN_IGNORE_ZONE = Color.ORANGE;
-    public static final Color BOLD_TEXT_IN_IGNORE_ZONE = Color.MAGENTA;
-
-    ViewerDocumentService viewerDocumentService;
+    static String ocrLayerName = "knecon OCR";
+    OcrServiceSettings settings;
 
 
     @SneakyThrows
-    public void drawOcrResultsToPdf(File document, File viewerDocument, Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
+    public Set<COSDictionary> drawOcrResultsToPdf(PDDocument document, Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
 
-        Map<Integer, VisualizationsOnPage> ocrVisualizationsOnPages = new HashMap<>();
-        Map<Integer, VisualizationsOnPage> ocrTextDebugVisualizationsOnPages = new HashMap<>();
-        Map<Integer, VisualizationsOnPage> ocrBBoxDebugVisualizationsOnPages = new HashMap<>();
-
-        try (var in = new FileInputStream(document); PDFDoc doc = new PDFDoc(in)) {
-
-            for (Integer pageNumber : imagesWithResultsPerPage.keySet()) {
-
-                List<Rectangle2D> textBBoxes = getTextBBoxes(doc.getPage(pageNumber));
-
-                ocrVisualizationsOnPages.put(pageNumber - 1, createVisualizations(imagesWithResultsPerPage.get(pageNumber), textBBoxes));
-                ocrTextDebugVisualizationsOnPages.put(pageNumber - 1, createDebugTextVisualizations(imagesWithResultsPerPage.get(pageNumber), textBBoxes));
-                ocrBBoxDebugVisualizationsOnPages.put(pageNumber - 1, createDebugBBoxVisualizations(imagesWithResultsPerPage.get(pageNumber)));
-            }
-        }
-
-        Visualizations visualizations = new Visualizations(ContentStreams.KNECON_OCR, ocrVisualizationsOnPages, false);
-
-        List<Visualizations> debugVisualizations = List.of(visualizations,
-                                                           new Visualizations(ContentStreams.KNECON_OCR_TEXT_DEBUG, ocrTextDebugVisualizationsOnPages, false),
-                                                           new Visualizations(ContentStreams.KNECON_OCR_BBOX_DEBUG, ocrBBoxDebugVisualizationsOnPages, false));
-
-        viewerDocumentService.addVisualizationsOnPage(document, document, List.of(visualizations));
-        viewerDocumentService.addVisualizationsOnPage(viewerDocument, viewerDocument, debugVisualizations);
-    }
-
-
-    private List<Rectangle2D> getTextBBoxes(Page page) {
-
-        List<Rectangle2D> textBBoxes = new ArrayList<>();
-        try (var textExtractor = new TextExtractor()) {
-            textExtractor.begin(page);
-
-            try {
-                for (TextExtractor.Line line = textExtractor.getFirstLine(); line.isValid(); line = line.getNextLine()) {
-                    for (var word = line.getFirstWord(); word.isValid(); word = word.getNextWord()) {
-                        textBBoxes.add(Converter.toRectangle2D(word.getBBox()));
-                    }
-                }
-            } catch (Exception e) {
-                log.warn("Could not get word dimension, {}", e.getMessage());
-            }
-            return textBBoxes;
-        }
-    }
-
-
-    private static Function<Integer, Integer> pageNumber1IdxTo0IdxMapper() {
-        // PDFBox uses a 0-based index for page numbers internally, while we use a 1-based index
-        return p -> p - 1;
-    }
-
-
-    private VisualizationsOnPage createVisualizations(List<OcrResultToWrite> ocrResultsToWrite, List<Rectangle2D> ignoreZones) {
-
-        List<TextPositionInImage> words = ocrResultsToWrite.stream()
-                .map(OcrResultToWrite::textPositionInImage)
-                .flatMap(Collection::stream)
-                .filter(word -> ignoreZones.stream()
-                        .noneMatch(ignoreZone -> word.getTransformedTextBBox().getBounds2D().intersects(ignoreZone)))
-                .toList();
-
-        List<PlacedText> placedTexts = words.stream()
-                .map(word -> new PlacedText(word.getText(),
-                                            null,
-                                            Color.BLACK,
-                                            (float) word.getFontSize(),
-                                            word.getFontMetricsFactory(),
-                                            Optional.of(word.getTextMatrix()),
-                                            Optional.of(RenderingMode.NEITHER)))
-                .toList();
-        return VisualizationsOnPage.builder().placedTexts(placedTexts).build();
-    }
-
-
-
-    private VisualizationsOnPage createDebugTextVisualizations(List<OcrResultToWrite> ocrResultsToWrite, List<Rectangle2D> textBBoxes) {
-
-        List<TextPositionInImage> wordsToDraw = new ArrayList<>();
-        List<TextPositionInImage> ignoredWords = new ArrayList<>();
-
-        for (OcrResultToWrite ocrResultToWrite : ocrResultsToWrite) {
-            for (TextPositionInImage textPositionInImage : ocrResultToWrite.textPositionInImage()) {
-                if (textBBoxes.stream()
-                        .anyMatch(ignoreZone -> textPositionInImage.getTransformedTextBBox().getBounds2D().intersects(ignoreZone))) {
-                    ignoredWords.add(textPositionInImage);
-                } else {
-                    wordsToDraw.add(textPositionInImage);
-                }
-            }
-        }
-        Stream<PlacedText> placedTexts = wordsToDraw.stream()
-                .map(word -> new PlacedText(word.getText(),
-                                            null,
-                                            word.getFontStyle().equals(FontStyle.REGULAR) ? REGULAR_TEXT_COLOR : BOLD_TEXT_COLOR,
-                                            (float) word.getFontSize(),
-                                            word.getFontMetricsFactory(),
-                                            Optional.of(word.getTextMatrix()),
-                                            Optional.of(RenderingMode.FILL)));
-
-        Stream<PlacedText> placedTexts2 = ignoredWords.stream()
-                .map(word -> new PlacedText(word.getText(),
-                                            null,
-                                            word.getFontStyle().equals(FontStyle.REGULAR) ? REGULAR_TEXT_IN_IGNORE_ZONE : BOLD_TEXT_IN_IGNORE_ZONE,
-                                            (float) word.getFontSize(),
-                                            word.getFontMetricsFactory(),
-                                            Optional.of(word.getTextMatrix()),
-                                            Optional.of(RenderingMode.FILL)));
-
-        return VisualizationsOnPage.builder()
-                .placedTexts(Stream.of(placedTexts, placedTexts2)
-                                     .flatMap(Function.identity())
-                                     .toList())
-                .build();
-    }
-
-
-
-    private VisualizationsOnPage createDebugBBoxVisualizations(List<OcrResultToWrite> ocrResultsToWrite) {
-
-        List<TextPositionInImage> words = ocrResultsToWrite.stream()
-                .map(OcrResultToWrite::textPositionInImage)
-                .flatMap(Collection::stream)
-                .toList();
-        List<ColoredLine> coloredLines = Stream.concat(//
-                                                       words.stream()
-                                                               .map(TextPositionInImage::getTransformedTextBBox)
-                                                               .map(this::quadPointAsLines),//
-                                                       ocrResultsToWrite.stream()
-                                                               .map(OcrResultToWrite::imageBoundingBox)
-                                                               .map(this::createGrid)//
-                )
-                .flatMap(Collection::stream)
-                .toList();
-        return VisualizationsOnPage.builder().coloredLines(coloredLines).build();
-    }
-
-
-    private List<ColoredLine> quadPointAsLines(QuadPoint rect) {
-
-        return List.of(new ColoredLine(new Line2D.Double(rect.a(), rect.b()), REGULAR_TEXT_IN_IGNORE_ZONE, 1),
-                       new ColoredLine(new Line2D.Double(rect.b(), rect.c()), REGULAR_TEXT_COLOR, 1),
-                       new ColoredLine(new Line2D.Double(rect.c(), rect.d()), Color.GREEN, 1),
-                       new ColoredLine(new Line2D.Double(rect.d(), rect.a()), BOLD_TEXT_IN_IGNORE_ZONE, 1));
+        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
+        imagesWithResultsPerPage.keySet().forEach(pageNumber -> drawResultsPerPage(document, pageNumber, imagesWithResultsPerPage.get(pageNumber), dictionariesToUpdate));
+        dictionariesToUpdate.add(document.getDocumentInformation().getCOSObject());
+        return dictionariesToUpdate;
     }
 
 
     @SneakyThrows
-    private List<ColoredLine> createGrid(QuadPoint rect) {
+    private void drawResultsPerPage(PDDocument document, Integer pageNumber, List<OcrResultToWrite> ocrResultToWrite, Set<COSDictionary> dictionariesToUpdate) {
 
-        List<ColoredLine> lines = new LinkedList<>(quadPointAsLines(rect));
+        var pdPage = document.getPage(pageNumber - 1);
 
+        PDOptionalContentGroup textDebugLayer = new PDOptionalContentGroup(ocrLayerName);
+        PDOptionalContentGroup bBoxDebugLayer = new PDOptionalContentGroup(ocrLayerName + "BBox");
+        if (settings.isDebug()) {
+            textDebugLayer = addOptionalGroup(ocrLayerName, document, pdPage, dictionariesToUpdate);
+            bBoxDebugLayer = addOptionalGroup(ocrLayerName + " BBox", document, pdPage, dictionariesToUpdate);
+        }
+
+        escapeContentStreams(document, pdPage);
+
+        List<TextPositionInImage> words = ocrResultToWrite.stream().map(OcrResultToWrite::textPositionInImage).flatMap(Collection::stream).toList();
+        try (var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true)) {
+
+            // write invisible ocr text inside tagged content
+            contentStream.beginMarkedContent(settings.getOcrMarkedContentTag());
+            contentStream.saveGraphicsState();
+            contentStream.setNonStrokingColor(Color.BLUE);
+            contentStream.setStrokingColor(Color.BLUE);
+            contentStream.setLineWidth(1);
+            words.forEach(word -> drawInvisibleWord(word, contentStream));
+            contentStream.restoreGraphicsState();
+            contentStream.endMarkedContent();
+
+            if (settings.isDebug()) { // must not be written, as it will interfere with layout parsing
+                // write visible ocr text inside optional group
+                contentStream.beginMarkedContent(COSName.OC, textDebugLayer);
+                contentStream.saveGraphicsState();
+                words.forEach(word -> drawVisibleWord(word, contentStream));
+                contentStream.restoreGraphicsState();
+                contentStream.endMarkedContent();
+
+                // write word bounding boxes (tesseract output) inside optional group
+                contentStream.beginMarkedContent(COSName.OC, bBoxDebugLayer);
+                contentStream.saveGraphicsState();
+                ocrResultToWrite.stream()
+                        .map(OcrResultToWrite::imageBoundingBox)
+                        .forEach(imagePosition -> drawGrid(contentStream, imagePosition));
+                words.stream().map(TextPositionInImage::getTransformedTextBBox).forEach(word -> drawRectangle(contentStream, word));
+                contentStream.restoreGraphicsState();
+                contentStream.endMarkedContent();
+            }
+        }
+        dictionariesToUpdate.add(pdPage.getCOSObject());
+        dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
+    }
+
+
+    @SneakyThrows
+    private static void escapeContentStreams(PDDocument document, PDPage pdPage) {
+        // We need to append to the contentstream, otherwise the content could be overlapped by images
+        // But we also need to save the graphics state before, such that our appended content cannot be affected by previous contentstreams with side-effects, such as not escaped matrix transformations
+        try (var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.PREPEND, false)) {
+            contentStream.saveGraphicsState();
+        }
+        try (var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, false)) {
+            contentStream.restoreGraphicsState();
+        }
+    }
+
+
+    private PDOptionalContentGroup addOptionalGroup(String ocrLayerName, PDDocument document, PDPage pdPage, Set<COSDictionary> dictionariesToUpdate) {
+
+        PDDocumentCatalog catalog = document.getDocumentCatalog();
+        PDOptionalContentProperties ocprops = catalog.getOCProperties();
+        if (ocprops == null) {
+            ocprops = new PDOptionalContentProperties();
+            catalog.setOCProperties(ocprops);
+        }
+        PDOptionalContentGroup layer = null;
+        if (ocprops.hasGroup(ocrLayerName)) {
+            layer = ocprops.getGroup(ocrLayerName);
+        } else {
+            layer = new PDOptionalContentGroup(ocrLayerName);
+            ocprops.addGroup(layer);
+        }
+
+        // enable debug layers by default only when DEBUG flag is set.
+        ocprops.setGroupEnabled(layer, settings.isDebug());
+        PDResources resources = pdPage.getResources();
+        if (resources == null) {
+            resources = new PDResources();
+            pdPage.setResources(resources);
+        }
+        dictionariesToUpdate.add(catalog.getCOSObject());
+        return layer;
+    }
+
+
+    @SneakyThrows
+    private void drawRectangle(PDPageContentStream contentStream, QuadPoint rect) {
+
+        contentStream.saveGraphicsState();
+        contentStream.setLineWidth(1);
+        contentStream.moveTo((float) rect.a().getX(), (float) rect.a().getY());
+        contentStream.lineTo((float) rect.b().getX(), (float) rect.b().getY());
+        contentStream.setStrokingColor(Color.ORANGE);
+        contentStream.stroke();
+        contentStream.moveTo((float) rect.b().getX(), (float) rect.b().getY());
+        contentStream.lineTo((float) rect.c().getX(), (float) rect.c().getY());
+        contentStream.setStrokingColor(Color.BLUE);
+        contentStream.stroke();
+        contentStream.moveTo((float) rect.c().getX(), (float) rect.c().getY());
+        contentStream.lineTo((float) rect.d().getX(), (float) rect.d().getY());
+        contentStream.setStrokingColor(Color.GREEN);
+        contentStream.stroke();
+        contentStream.moveTo((float) rect.d().getX(), (float) rect.d().getY());
+        contentStream.lineTo((float) rect.a().getX(), (float) rect.a().getY());
+        contentStream.setStrokingColor(Color.MAGENTA);
+        contentStream.stroke();
+        contentStream.restoreGraphicsState();
+    }
+
+
+    private void drawInvisibleWord(TextPositionInImage word, PDPageContentStream contentStream) {
+
+        drawWord(word, contentStream, RenderingMode.NEITHER);
+    }
+
+
+    private void drawVisibleWord(TextPositionInImage word, PDPageContentStream contentStream) {
+
+        drawWord(word, contentStream, RenderingMode.FILL);
+    }
+
+
+    //    @SneakyThrows
+    private void drawWord(TextPositionInImage position, PDPageContentStream contentStream, RenderingMode renderingMode) {
+
+        try {
+            contentStream.setNonStrokingColor(switch (position.getFontStyle()) {
+                case BOLD -> Color.RED;
+                case ITALIC -> Color.GREEN;
+                default -> Color.BLUE;
+            });
+            contentStream.beginText();
+            contentStream.setRenderingMode(renderingMode);
+            contentStream.setFont(position.getFont(), (float) position.getFontSize());
+            contentStream.setTextMatrix(position.getTextMatrix());
+            contentStream.showText(position.getText());
+            contentStream.endText();
+
+        } catch (Exception e) {
+            log.error("Failed to write text {}", position.getText());
+            log.error(e.getMessage());
+        }
+    }
+
+
+    @SneakyThrows
+    private void drawGrid(PDPageContentStream contentStream, QuadPoint rect) {
+
+        drawRectangle(contentStream, rect);
+
+        contentStream.saveGraphicsState();
+        contentStream.setStrokingColor(Color.BLACK);
+        contentStream.setLineWidth(0.2F);
         int nRows = 8;
         int nCols = 8;
 
@@ -218,7 +218,7 @@ public class OcrResultWriter {
         Point2D start = add(rect.a(), abStep);
         Point2D end = add(rect.d(), abStep);
         for (int row = 0; row < nRows; ++row) {
-            lines.add(new ColoredLine(new Line2D.Double(start, end), Color.BLACK, 0.2f));
+            drawLine(start, end, contentStream);
             start = add(start, abStep);
             end = add(end, abStep);
         }
@@ -226,12 +226,21 @@ public class OcrResultWriter {
         start = add(rect.a(), adStep);
         end = add(rect.b(), adStep);
         for (int col = 0; col < nCols; ++col) {
-            lines.add(new ColoredLine(new Line2D.Double(start, end), Color.BLACK, 0.2f));
+            drawLine(start, end, contentStream);
             start = add(start, adStep);
             end = add(end, adStep);
         }
+        contentStream.restoreGraphicsState();
 
-        return lines;
+    }
+
+
+    @SneakyThrows
+    private void drawLine(Point2D a, Point2D b, PDPageContentStream contentStream) {
+
+        contentStream.moveTo((float) a.getX(), (float) a.getY());
+        contentStream.lineTo((float) b.getX(), (float) b.getY());
+        contentStream.stroke();
     }
 
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/fonts/FontMetricsFactory.java

@@ -4,12 +4,11 @@ import org.apache.pdfbox.pdmodel.font.PDFont;
 
 import com.knecon.fforesight.service.ocr.processor.model.FontMetrics;
 import com.knecon.fforesight.service.ocr.processor.model.HeightAndDescent;
-import com.knecon.fforesight.service.viewerdoc.model.EmbeddableFont;
 
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
-public interface FontMetricsFactory extends EmbeddableFont {
+public interface FontMetricsFactory {
 
     default FontMetrics calculateMetrics(String text, double textWidth, double textHeight) {
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/fonts/Type0FontMetricsFactory.java

@@ -1,6 +1,8 @@
 package com.knecon.fforesight.service.ocr.processor.service.fonts;
 
 import java.io.ByteArrayInputStream;
+import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 
 import org.apache.fontbox.ttf.GlyphData;
@@ -13,63 +15,45 @@ import org.apache.pdfbox.pdmodel.font.PDType0Font;
 
 import com.knecon.fforesight.service.ocr.processor.model.HeightAndDescent;
 
-import lombok.AllArgsConstructor;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import software.amazon.awssdk.services.s3.endpoints.internal.Value;
 
 @Slf4j
 @RequiredArgsConstructor
-@AllArgsConstructor
 public class Type0FontMetricsFactory implements FontMetricsFactory {
 
-    private final String resourcePath;
-    private PDType0Font type0Font;
-    private TrueTypeFont trueTypeFont;
-    private PDDocument documentThisIsEmbeddedIn;
+    private final PDType0Font type0Font;
+    private final TrueTypeFont trueTypeFont;
 
     // for this specific font back-/forward-slashes have a lot of descent screwing up the font size and therefore bold detection. So if we find such a character we ignore its descent.
     private static final Set<Integer> slashGlyphIds = Set.of(18, 63);
 
 
-    @SneakyThrows
     public static Type0FontMetricsFactory regular(PDDocument document) {
 
-        String resourcePath = "fonts/cmu-regular.ttf";
-        return createFromResourcePath(resourcePath, document);
+        return createFromResource("fonts/cmu-regular.ttf", document);
     }
 
 
-    @SneakyThrows
     public static Type0FontMetricsFactory bold(PDDocument document) {
 
-        String resourcePath = "fonts/cmu-bold.ttf";
-        return createFromResourcePath(resourcePath, document);
+        return createFromResource("fonts/cmu-bold.ttf", document);
     }
 
 
     @SneakyThrows
-    @SuppressWarnings("PMD.CloseResource")
-    private static TrueTypeFont readFromResourcePath(String resourcePath) {
+    private static Type0FontMetricsFactory createFromResource(String resourcePath, PDDocument document) {
 
-        // The ttf is closed with the document, see PDType0Font line 134
         try (var in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourcePath); var buffer = new RandomAccessReadBuffer(in)) {
-            return new TTFParser().parse(buffer);
+            TrueTypeFont trueTypeFont = new TTFParser().parse(buffer); // since Type0Font can be descendant from any font, we need to remember the original TrueTypeFont for the glyph information
+            PDType0Font type0Font = PDType0Font.load(document, trueTypeFont, true); // use Type0Font for unicode support
+            return new Type0FontMetricsFactory(type0Font, trueTypeFont);
         }
     }
 
 
-    @SneakyThrows
-    @SuppressWarnings("PMD.CloseResource")
-    private static Type0FontMetricsFactory createFromResourcePath(String resourcePath, PDDocument document) {
-
-        TrueTypeFont trueTypeFont = readFromResourcePath(resourcePath);
-        // since Type0Font can be descendant from any font, we need to remember the original TrueTypeFont for the glyph information
-        return new Type0FontMetricsFactory(resourcePath, PDType0Font.load(document, trueTypeFont, true), trueTypeFont, document); // use Type0Font for unicode support)
-
-    }
-
-
     @SneakyThrows
     public HeightAndDescent calculateHeightAndDescent(String text) {
 
@@ -113,28 +97,4 @@ public class Type0FontMetricsFactory implements FontMetricsFactory {
         return type0Font;
     }
 
-
-    @Override
-    @SneakyThrows
-    public PDFont embed(PDDocument document) {
-
-        if (documentThisIsEmbeddedIn.equals(document)) {
-            return getFont();
-        }
-
-        // no need to close, the font will be closed with the document it is embedded in
-
-        this.trueTypeFont = readFromResourcePath(resourcePath);
-        this.type0Font = PDType0Font.load(document, trueTypeFont, true);
-        this.documentThisIsEmbeddedIn = document;
-        return getFont();
-    }
-
-
-    @SneakyThrows
-    public void close() {
-
-        trueTypeFont.close();
-    }
-
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/scriptdetection/FontStyleDetector.java

@@ -49,11 +49,11 @@ public class FontStyleDetector {
      * (<a href="https://en.wikipedia.org/wiki/Opening_(morphology)">Opening (Morphology)</a>).
      * We then threshold the ratio of remaining pixels to determine whether a word is bold or not.
      * <p>
-     * I did take some liberties though. Firstly, the paper uses text height without ascender/descender height for the clustering. I'm using the previously implemented font size estimation.
-     * But that is calculated based on text width. Thus, I'm also using the height scaling factor to scale the font size by the text height.
+     * I did take some liberties though. Firstly, the paper uses text height without ascender/descender height for the clustering. I'm using the previously implemented font size.
+     * But this is based on text width. Thus, I'm also using the height scaling factor to scale the font size by the text height.
      * The paper does not describe its clustering algorithm, so I've decided on DBSCAN due to its good runtime and readily available implementation by apache commons math.
      * Moreover, the paper states that stroke width scales linearly with text height. I've come to the conclusion this is not the case.
-     * It seems it scales with the square root of the text height. Or at least this seemed to give the best results for me.
+     * It seems it scales with the square root of the text height. Or at least this seemed to give the best results.
      */
     public Map<Integer, List<OcrResultToWrite>> detectBold(List<OcrResult> ocrResults, PDDocument document) {
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/threads/GhostScriptOutputHandler.java

@@ -109,6 +109,7 @@ public class GhostScriptOutputHandler extends Thread {
         if (imageFile == null) {
             throw new IllegalArgumentException(String.format("Page number %d does not exist in this thread. It only has pagenumbers %s", pageNumber, pagesToProcess.keySet()));
         }
+        assert new File(imageFile.absoluteFilePath()).isFile();
         renderedPageImageFileOutput.add(imageFile);
     }
 

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/threads/ImageProcessingThread.java

@@ -45,7 +45,7 @@ public class ImageProcessingThread extends Thread {
     final BlockingQueue<UnprocessedImage> imageInputQueue;
     final BlockingQueue<OcrImage> imageOutputQueue;
     final ITessAPI.TessBaseAPI detectionScriptHandle = initDetectionScriptHandle();
-    final L_Kernel gaussianKernel = Leptonica1.makeGaussianKernel(2, 2, 0.7f, 1);
+    final L_Kernel gaussianKernel = Leptonica1.makeGaussianKernel(2, 2, 1.2f, 1);
     final Statistics stats;
     final OcrServiceSettings settings;
     final PDDocument document;
@@ -107,7 +107,6 @@ public class ImageProcessingThread extends Thread {
     }
 
 
-    @SuppressWarnings("PMD.CompareObjectsWithEquals")
     private OcrImage processRenderedPageImageFile(RenderedPageImageFile renderedPageImageFile) {
 
         Pix pix = processPix(renderedPageImageFile.asPix(), settings.getDpi(), settings.getDpi());
@@ -128,7 +127,7 @@ public class ImageProcessingThread extends Thread {
         return ocrImage;
     }
 
-    @SuppressWarnings("PMD.CompareObjectsWithEquals")
+
     private OcrImage processExtractedImage(ExtractedImage extractedImage) {
 
         float imageDPI = Math.abs(extractedImage.image().getWidth() / (extractedImage.ctm().getScalingFactorX() / 72));
@@ -199,10 +198,8 @@ public class ImageProcessingThread extends Thread {
             grayScale = pix;
         } else if (pix.d == 32) {
             grayScale = Leptonica1.pixConvertRGBToGrayFast(pix);
-            LeptUtils.disposePix(pix);
         } else if (pix.d == 1) {
             grayScale = Leptonica1.pixConvert1To8(null, pix, (byte) 0, (byte) 255);
-            LeptUtils.disposePix(pix);
         } else {
             throw new UnsupportedOperationException(String.format("Unknown pix format with bpp of %d", pix.d));
         }
@@ -211,27 +208,29 @@ public class ImageProcessingThread extends Thread {
         float targetFactor = targetDpi / imageDpi;
         if (targetFactor > 2.1) {
             scaledUp = Leptonica1.pixScaleGray4xLI(grayScale);
-            LeptUtils.disposePix(grayScale);
         } else if (targetFactor > 1.1) {
             scaledUp = Leptonica1.pixScaleGray2xLI(grayScale);
-            LeptUtils.disposePix(grayScale);
         } else {
             scaledUp = grayScale;
         }
 
         // remove noise and prep for Otsu
         gaussian = Leptonica1.pixConvolve(scaledUp, gaussianKernel, 8, 1);
-        LeptUtils.disposePix(scaledUp);
 
         // Threshold to binary
         if (pix.w < 100 || pix.h < 100) {
             binarized = Leptonica1.pixThresholdToBinary(gaussian, 170);
         } else {
-            binarized = Leptonica1.pixOtsuThreshOnBackgroundNorm(gaussian, null, 50, 50, 165, 10, 100, 5, 5, 0.0f, null);
+            binarized = Leptonica1.pixOtsuThreshOnBackgroundNorm(gaussian, null, 50, 50, 165, 10, 100, 5, 5, 0.2f, null);
+
             if (binarized == null) { // Sometimes Otsu just fails, then we binarize directly
                 binarized = Leptonica1.pixThresholdToBinary(gaussian, 170);
             }
         }
+
+        LeptUtils.disposePix(pix);
+        LeptUtils.disposePix(grayScale);
+        LeptUtils.disposePix(scaledUp);
         LeptUtils.disposePix(gaussian);
 
         return binarized;

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/settings/OcrServiceSettings.java

@@ -21,8 +21,10 @@ public class OcrServiceSettings {
     int minImageWidth = 20; // Minimum width for images to be processed
     float minRotationConfidence = 2; // Sets a lower bound for the confidence rating for rotated pages.
     boolean debug; // If true, overlays OCR images with a grid and draws word bounding boxes
+    boolean removeWatermark; // If true, watermarks will be removed
     String languages = "deu+eng"; // Defines languages loaded into Tesseract as 3-char codes, additional languages must also be installed in the docker environment
     COSName ocrMarkedContentTag = COSName.getPDFName("KNECON_OCR");
     boolean boldDetection = true; // if true, bold detection will be attempted
     double boldThreshold = 0.5; // Words are opened with a brick of average stroke width, if the ratio of remaining pixels is higher the word is determined bold.
+
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/utils/PdfDraw.java

@@ -17,57 +17,58 @@ public class PdfDraw {
     @SneakyThrows
     public static void drawGrid(ElementWriter writer, Page page) {
 
-        try (var eb = new ElementBuilder()) {
-            double dX = 15;
-            double dY = 15;
-            int nRows = (int) (page.getPageHeight() / dY) + 1;
-            int nCols = (int) (page.getPageWidth() / dX) + 1;
-            for (int row = 0; row < nRows; ++row) {
-                for (int col = 0; col < nCols; ++col) {
-                    Element cell = eb.createRect(col * dX, row * dY, dX, dY);
-                    cell.setPathStroke(true);
-                    cell.getGState().setLineWidth(1);
-                    cell.getGState().setStrokeOpacity(0.1);
-                    cell.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
-                    if (row == 0 && col == 0) {
-                        cell.getGState().setStrokeColor(new ColorPt(0, 0, 1));
-                        cell.setPathFill(true);
-                        cell.getGState().setFillOpacity(0.8);
-                        cell.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
-                        cell.getGState().setFillColor(new ColorPt(0, 0, 1));
-                    } else {
-                        cell.setPathFill(false);
-                        cell.getGState().setStrokeColor(new ColorPt(0.1, 0.1, 0.1));
-                    }
-                    writer.writePlacedElement(cell);
+        ElementBuilder eb = new ElementBuilder();
+        double dX = 15;
+        double dY = 15;
+        int nRows = (int) (page.getPageHeight() / dY) + 1;
+        int nCols = (int) (page.getPageWidth() / dX) + 1;
+        for (int row = 0; row < nRows; ++row) {
+            for (int col = 0; col < nCols; ++col) {
+                Element cell = eb.createRect(col * dX, row * dY, dX, dY);
+                cell.setPathStroke(true);
+                cell.getGState().setLineWidth(1);
+                cell.getGState().setStrokeOpacity(0.1);
+                cell.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+                if (row == 0 && col == 0) {
+                    cell.getGState().setStrokeColor(new ColorPt(0, 0, 1));
+                    cell.setPathFill(true);
+                    cell.getGState().setFillOpacity(0.8);
+                    cell.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
+                    cell.getGState().setFillColor(new ColorPt(0, 0, 1));
+                } else {
+                    cell.setPathFill(false);
+                    cell.getGState().setStrokeColor(new ColorPt(0.1, 0.1, 0.1));
                 }
+                writer.writePlacedElement(cell);
             }
         }
+        eb.destroy();
     }
 
 
     @SneakyThrows
     public static void drawRectCollection(ElementWriter writer, RectCollection rectCollection) {
 
-        try (var colorPt = new ColorPt(1, 0, 0); var eb = new ElementBuilder()) {
-            for (int i = 0; i < rectCollection.getNumRects(); ++i) {
-                try(var r = rectCollection.getRectAt(i)) {
-                    Element rect = eb.createRect(r.getX1(), r.getY1(), r.getWidth(), r.getHeight());
+        ColorPt colorPt = new ColorPt(1, 0, 0);
+        ElementBuilder eb = new ElementBuilder();
+        for (int i = 0; i < rectCollection.getNumRects(); ++i) {
+            Rect r = rectCollection.getRectAt(i);
+            Element rect = eb.createRect(r.getX1(), r.getY1(), r.getWidth(), r.getHeight());
 
-                    rect.setPathStroke(true);
-                    rect.getGState().setLineWidth(5);
-                    rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
-                    rect.getGState().setStrokeColor(colorPt);
+            rect.setPathStroke(true);
+            rect.getGState().setLineWidth(5);
+            rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+            rect.getGState().setStrokeColor(colorPt);
 
-                    rect.setPathFill(true);
-                    rect.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
-                    rect.getGState().setFillColor(colorPt);
-                    rect.getGState().setFillOpacity(0.5);
+            rect.setPathFill(true);
+            rect.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
+            rect.getGState().setFillColor(colorPt);
+            rect.getGState().setFillOpacity(0.5);
 
-                    writer.writePlacedElement(rect);
-                }
-            }
+            writer.writePlacedElement(rect);
         }
+        colorPt.destroy();
+        eb.destroy();
     }
 
 }

ocr-service-v1/ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/utils/Tesseract2.java

@@ -1,5 +1,25 @@
 package com.knecon.fforesight.service.ocr.processor.utils;
 
+import static net.sourceforge.tess4j.ITessAPI.TRUE;
+import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIDelete;
+import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIEnd;
+import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIGetIterator;
+import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIGetStringVariable;
+import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIMeanTextConf;
+import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIProcessPage;
+import static net.sourceforge.tess4j.TessAPI1.TessDeleteResultRenderer;
+import static net.sourceforge.tess4j.TessAPI1.TessHOcrRendererCreate;
+import static net.sourceforge.tess4j.TessAPI1.TessPageIteratorBegin;
+import static net.sourceforge.tess4j.TessAPI1.TessPageIteratorBoundingBox;
+import static net.sourceforge.tess4j.TessAPI1.TessPageIteratorNext;
+import static net.sourceforge.tess4j.TessAPI1.TessResultIteratorConfidence;
+import static net.sourceforge.tess4j.TessAPI1.TessResultIteratorDelete;
+import static net.sourceforge.tess4j.TessAPI1.TessResultIteratorGetPageIterator;
+import static net.sourceforge.tess4j.TessAPI1.TessResultIteratorGetUTF8Text;
+import static net.sourceforge.tess4j.TessAPI1.TessResultRendererBeginDocument;
+import static net.sourceforge.tess4j.TessAPI1.TessResultRendererEndDocument;
+import static net.sourceforge.tess4j.TessAPI1.TessResultRendererInsert;
+
 import java.awt.Rectangle;
 import java.nio.IntBuffer;
 import java.util.ArrayList;
@@ -9,20 +29,19 @@ import com.sun.jna.Pointer;
 
 import lombok.extern.slf4j.Slf4j;
 import net.sourceforge.lept4j.Pix;
+import net.sourceforge.tess4j.ITessAPI;
 import net.sourceforge.tess4j.OCRResult;
 import net.sourceforge.tess4j.TessAPI1;
-import net.sourceforge.tess4j.Tesseract1;
+import net.sourceforge.tess4j.Tesseract;
 import net.sourceforge.tess4j.TesseractException;
 import net.sourceforge.tess4j.Word;
 
 @Slf4j
 /**
  * Overriden version only so I can use Tesseract1 with Pixs instead of BufferedImages. All Functions are copied and then the BufferedImage -> Pix conversion deleted.
- */
-public class Tesseract2 extends Tesseract1 {
+ */ public class Tesseract2 extends Tesseract {
 
-
-    private int createDocuments(Pix pix, String filename, TessResultRenderer renderer) {
+    private int createDocuments(Pix pix, String filename, ITessAPI.TessResultRenderer renderer) {
 
         String title = TessBaseAPIGetStringVariable(getHandle(), DOCUMENT_TITLE);
         TessResultRendererBeginDocument(renderer, title);
@@ -62,7 +81,7 @@ public class Tesseract2 extends Tesseract1 {
         try {
             for (int i = 0; i < pixs.length; i++) {
                 try {
-                    TessResultRenderer renderer = createRenderers(outputbases[i], formats);
+                    ITessAPI.TessResultRenderer renderer = createRenderers(outputbases[i], formats);
                     int meanTextConfidence = createDocuments(pixs[i], filenames[i], renderer);
                     TessDeleteResultRenderer(renderer);
                     List<Word> words = meanTextConfidence > 0 ? getRecognizedWords(pageIteratorLevel) : new ArrayList<Word>();
@@ -85,8 +104,8 @@ public class Tesseract2 extends Tesseract1 {
         List<Word> words = new ArrayList<>();
 
         try {
-            TessResultIterator ri = TessBaseAPIGetIterator(getHandle());
-            TessPageIterator pi = TessResultIteratorGetPageIterator(ri);
+            ITessAPI.TessResultIterator ri = TessBaseAPIGetIterator(getHandle());
+            ITessAPI.TessPageIterator pi = TessResultIteratorGetPageIterator(ri);
             TessPageIteratorBegin(pi);
 
             do {
@@ -119,9 +138,9 @@ public class Tesseract2 extends Tesseract1 {
     }
 
 
-    private TessResultRenderer createRenderers(String outputbase, List<RenderedFormat> formats) {
+    private ITessAPI.TessResultRenderer createRenderers(String outputbase, List<RenderedFormat> formats) {
 
-        TessResultRenderer renderer = null;
+        ITessAPI.TessResultRenderer renderer = null;
 
         for (RenderedFormat format : formats) {
             switch (format) {
@@ -138,4 +157,12 @@ public class Tesseract2 extends Tesseract1 {
         return renderer;
     }
 
+
+    @Override
+    protected void dispose() {
+
+        TessBaseAPIEnd(getHandle());
+        TessBaseAPIDelete(getHandle());
+    }
+
 }

ocr-service-v1/ocr-service-server/build.gradle.kts

@@ -6,7 +6,7 @@ plugins {
     id("org.springframework.boot") version "3.1.5"
     id("io.spring.dependency-management") version "1.1.3"
     id("org.sonarqube") version "4.3.0.3225"
-    id("io.freefair.lombok") version "8.4"
+    id("io.freefair.lombok") version "8.2.2"
 }
 
 configurations {
@@ -23,7 +23,7 @@ dependencies {
     implementation(project(":ocr-service-processor"))
     implementation(project(":ocr-service-api"))
 
-    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("com.knecon.fforesight:tracing-commons:0.3.0")
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
     implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
 

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/Application.java

@@ -5,6 +5,7 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.annotation.EnableAsync;
@@ -12,6 +13,7 @@ import org.springframework.scheduling.annotation.EnableAsync;
 import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
 import com.iqser.red.pdftronlogic.commons.WatermarkRemovalService;
 import com.knecon.fforesight.service.ocr.processor.OcrServiceProcessorConfiguration;
+import com.knecon.fforesight.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.knecon.fforesight.service.ocr.v1.server.queue.MessagingConfiguration;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@@ -23,6 +25,7 @@ import io.micrometer.core.instrument.MeterRegistry;
 @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 @Import({MessagingConfiguration.class, StorageAutoConfiguration.class, OcrServiceProcessorConfiguration.class})
+@EnableFeignClients(basePackageClasses = FileStatusProcessingUpdateClient.class)
 public class Application {
 
     /**

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/client/FileStatusProcessingUpdateClient.java

@@ -0,0 +1,10 @@
+package com.knecon.fforesight.service.ocr.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;
+
+@FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
+public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {
+
+}

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/MessagingConfiguration.java

@@ -11,10 +11,35 @@ import lombok.RequiredArgsConstructor;
 @RequiredArgsConstructor
 public class MessagingConfiguration {
 
-    public static final String OCR_REQUEST_QUEUE = "ocr_request_queue";
-    public static final String OCR_RESPONSE_QUEUE = "ocr_response_queue";
+    public static final String OCR_QUEUE = "ocrQueue";
+    public static final String OCR_DLQ = "ocrDLQ";
+
+    public static final String X_DEAD_LETTER_EXCHANGE = "x-dead-letter-exchange";
+    public static final String X_DEAD_LETTER_ROUTING_KEY = "x-dead-letter-routing-key";
+    public static final String X_MAX_PRIORITY = "x-max-priority";
+
     public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue";
+
     public static final String X_ERROR_INFO_HEADER = "x-error-message";
     public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
 
+
+    @Bean
+    public Queue ocrQueue() {
+
+        return QueueBuilder.durable(OCR_QUEUE)
+                .withArgument(X_DEAD_LETTER_EXCHANGE, "")
+                .withArgument(X_DEAD_LETTER_ROUTING_KEY, OCR_DLQ)
+                .withArgument(X_MAX_PRIORITY, 2)
+                .maxPriority(2)
+                .build();
+    }
+
+
+    @Bean
+    public Queue ocrDeadLetterQueue() {
+
+        return QueueBuilder.durable(OCR_DLQ).build();
+    }
+
 }

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/NoStatusUpdateOcrMessageSender.java

@@ -1,42 +0,0 @@
-package com.knecon.fforesight.service.ocr.v1.server.queue;
-
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.ocr.processor.service.IOcrMessageSender;
-import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@ConditionalOnProperty(value = "ocrService.sendStatusUpdates", havingValue = "false")
-public class NoStatusUpdateOcrMessageSender implements IOcrMessageSender {
-
-    RabbitTemplate rabbitTemplate;
-
-
-    public void sendOcrFinished(String fileId, int totalImages) {
-
-    }
-
-
-    public void sendOCRStarted(String fileId) {
-
-    }
-
-
-    public void sendUpdate(String fileId, int finishedImages, int totalImages) {
-
-    }
-
-
-    public void sendOcrResponse(String dossierId, String fileId) {
-
-        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_RESPONSE_QUEUE, new DocumentRequest(dossierId, fileId));
-    }
-
-}

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/OcrMessageReceiver.java

@@ -1,8 +1,8 @@
 package com.knecon.fforesight.service.ocr.v1.server.queue;
 
-import java.io.File;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 
@@ -10,15 +10,17 @@ import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Service;
-import org.springframework.util.FileSystemUtils;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.knecon.fforesight.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.knecon.fforesight.service.ocr.processor.service.FileStorageService;
-import com.knecon.fforesight.service.ocr.processor.service.IOcrMessageSender;
 import com.knecon.fforesight.service.ocr.processor.service.OCRService;
-import com.knecon.fforesight.service.ocr.processor.service.OsUtils;
 import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
+
+import feign.FeignException;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -30,49 +32,71 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class OcrMessageReceiver {
 
-    FileStorageService fileStorageService;
-    ObjectMapper objectMapper;
-    OCRService ocrService;
-    IOcrMessageSender ocrMessageSender;
+     FileStorageService fileStorageService;
+     ObjectMapper objectMapper;
+     FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
+     OCRService ocrService;
 
 
     @RabbitHandler
-    @RabbitListener(queues = MessagingConfiguration.OCR_REQUEST_QUEUE, concurrency = "1")
+    @RabbitListener(queues = MessagingConfiguration.OCR_QUEUE, concurrency = "1")
     public void receiveOcr(Message in) throws IOException {
 
-        if (in.getMessageProperties().isRedelivered()) {
-            throw new AmqpRejectAndDontRequeueException("Redelivered OCR Request, aborting...");
-        }
-
         DocumentRequest ocrRequestMessage = objectMapper.readValue(in.getBody(), DocumentRequest.class);
-        String dossierId = ocrRequestMessage.getDossierId();
-        String fileId = ocrRequestMessage.getFileId();
-        Path tmpDir = Path.of(OsUtils.getTemporaryDirectory()).resolve(dossierId + "-" + fileId);
+        log.info("--------------------------------------------------------------------------");
+        log.info("Start ocr for file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
 
         try {
-            log.info("--------------------------------------------------------------------------");
-            log.info("Start ocr for file with dossierId {} and fileId {}", dossierId, fileId);
+            setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
 
-            ocrMessageSender.sendOCRStarted(fileId);
+            if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) {
+                byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+                fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile);
+            }
 
-            tmpDir.toFile().mkdirs();
-            File documentFile = tmpDir.resolve("document.pdf").toFile();
-            File viewerDocumentFile = tmpDir.resolve("viewerDocument.pdf").toFile();
+            try (var transferStream = new ByteArrayOutputStream()) {
+                ocrService.runOcrOnDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), transferStream);
+                try (var inputStream = new ByteArrayInputStream(transferStream.toByteArray())) {
+                    fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), inputStream);
+                }
+            } catch (IOException e) {
+                log.error("Failed to store file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+                throw new RuntimeException(e);
+            }
 
-            fileStorageService.downloadFiles(dossierId, fileId, documentFile, viewerDocumentFile);
-
-            ocrService.runOcrOnDocument(dossierId, fileId, ocrRequestMessage.isRemoveWatermark(), tmpDir, documentFile, viewerDocumentFile);
-
-            fileStorageService.storeFiles(dossierId, fileId, documentFile, viewerDocumentFile);
-
-            ocrMessageSender.sendOcrResponse(dossierId, fileId);
+            fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
         } catch (Exception e) {
             log.warn("An exception occurred in ocr file stage: {}", e.getMessage());
             in.getMessageProperties().getHeaders().put(MessagingConfiguration.X_ERROR_INFO_HEADER, e.getMessage());
             in.getMessageProperties().getHeaders().put(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
             throw new RuntimeException(e);
-        } finally {
-            FileSystemUtils.deleteRecursively(tmpDir);
+        }
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = MessagingConfiguration.OCR_DLQ, concurrency = "1")
+    public void receiveOcrDLQ(Message failedMessage) throws IOException {
+
+        DocumentRequest ocrRequestMessage = objectMapper.readValue(failedMessage.getBody(), DocumentRequest.class);
+        log.info("OCR DQL received: {}", ocrRequestMessage);
+        String errorMessage = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER);
+        OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER);
+        timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
+        fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(),
+                ocrRequestMessage.getFileId(),
+                new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp));
+    }
+
+
+    private void setStatusOcrProcessing(String dossierId, String fileId) {
+
+        try {
+            fileStatusProcessingUpdateClient.ocrProcessing(dossierId, fileId);
+        } catch (FeignException e) {
+            if (e.status() == HttpStatus.CONFLICT.value()) {
+                throw new AmqpRejectAndDontRequeueException(e.getMessage());
+            }
         }
     }
 

ocr-service-v1/ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/OcrMessageSender.java

@@ -1,23 +1,17 @@
 package com.knecon.fforesight.service.ocr.v1.server.queue;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.ocr.processor.service.IOcrMessageSender;
-import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
 import com.knecon.fforesight.service.ocr.v1.api.model.OCRStatusUpdateResponse;
 
-import jakarta.annotation.PostConstruct;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
 
-@Slf4j
 @Service
 @RequiredArgsConstructor
-@ConditionalOnProperty(value = "ocrService.sendStatusUpdates", havingValue = "true")
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class OcrMessageSender implements IOcrMessageSender {
 
@@ -31,14 +25,6 @@ public class OcrMessageSender implements IOcrMessageSender {
     }
 
 
-    public void sendOCRStarted(String fileId) {
-
-        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
-                OCRStatusUpdateResponse.builder().fileId(fileId).ocrStarted(true).build());
-
-    }
-
-
     public void sendUpdate(String fileId, int finishedImages, int totalImages) {
 
         rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
@@ -46,10 +32,4 @@ public class OcrMessageSender implements IOcrMessageSender {
 
     }
 
-
-    public void sendOcrResponse(String dossierId, String fileId) {
-
-        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_RESPONSE_QUEUE, new DocumentRequest(dossierId, fileId));
-    }
-
 }

ocr-service-v1/ocr-service-server/src/main/resources/application.yml

@@ -57,6 +57,3 @@ management:
       endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}
 
 pdftron.license: ${PDFTRON_LICENSE}
-
-ocrService:
-  sendStatusUpdates: true

ocr-service-v1/ocr-service-server/src/main/resources/vcpkg.json

@@ -6,7 +6,7 @@
   "overrides": [
     {
       "name": "tesseract",
-      "version": "5.3.4"
+      "version": "5.3.2"
     },
     {
       "name": "leptonica",

ocr-service-v1/ocr-service-server/src/test/java/com/knecon/fforesight/service/ocr/v1/server/AbstractTest.java

@@ -24,11 +24,10 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.knecon.fforesight.service.ocr.processor.initializer.NativeLibrariesInitializer;
+import com.knecon.fforesight.service.ocr.processor.initializer.PDFNetInitializer;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
-import com.knecon.fforesight.service.ocr.v1.server.queue.OcrMessageSender;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 import com.pdftron.pdf.PDFNet;
 
@@ -37,7 +36,7 @@ import lombok.SneakyThrows;
 
 @ExtendWith({SpringExtension.class, MockitoExtension.class})
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import({AbstractTest.TestConfiguration.class, NativeLibrariesInitializer.class})
+@Import({AbstractTest.TestConfiguration.class, PDFNetInitializer.class})
 @AutoConfigureObservability
 public class AbstractTest {
 
@@ -47,9 +46,6 @@ public class AbstractTest {
     @MockBean
     private TenantsClient tenantsClient;
 
-    @MockBean
-    private OcrMessageSender ocrMessageSender;
-
     @Autowired
     protected StorageService storageService;
 

ocr-service-v1/ocr-service-server/src/test/java/com/knecon/fforesight/service/ocr/v1/server/OcrServiceIntegrationTest.java

@@ -9,47 +9,29 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.core.io.ClassPathResource;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.pdftronlogic.commons.ClippingPathStack;
-import com.iqser.red.pdftronlogic.commons.ElementFeatures;
-import com.iqser.red.pdftronlogic.commons.MarkedContentStack;
+import com.knecon.fforesight.service.ocr.processor.service.FileStorageService;
 import com.knecon.fforesight.service.ocr.processor.service.OCRService;
-import com.knecon.fforesight.service.ocr.processor.service.OsUtils;
-import com.pdftron.common.PDFNetException;
-import com.pdftron.pdf.ColorPt;
-import com.pdftron.pdf.ColorSpace;
-import com.pdftron.pdf.Element;
-import com.pdftron.pdf.ElementReader;
-import com.pdftron.pdf.ElementWriter;
-import com.pdftron.pdf.GState;
-import com.pdftron.pdf.PDFDoc;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.PageIterator;
-import com.pdftron.pdf.Rect;
-import com.pdftron.sdf.Obj;
-import com.pdftron.sdf.SDFDoc;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.prometheus.PrometheusMeterRegistry;
 import io.micrometer.prometheus.PrometheusTimer;
-import lombok.Builder;
 import lombok.SneakyThrows;
 
-//@Disabled // Ghostscript/Tesseract/Leptonica is not available on build server. If you want to run the test install these dependencies. See README.md for help.
+@Disabled // Ghostscript/Tesseract/Leptonica is not available on build server. If you want to run the test install these dependencies. See README.md for help.
 @SpringBootTest()
 public class OcrServiceIntegrationTest extends AbstractTest {
 
@@ -70,10 +52,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
         testOCR("files/Watermark.pdf");
         testOCR("files/Watermark.pdf");
 
-        var ocrOnDocumentMeter = registry.getMeters()
-                .stream()
-                .filter(m -> m.getId().getName().equalsIgnoreCase("redactmanager_runOcrOnDocument"))
-                .findAny();
+        var ocrOnDocumentMeter = registry.getMeters().stream().filter(m -> m.getId().getName().equalsIgnoreCase("redactmanager_runOcrOnDocument")).findAny();
         assertThat(ocrOnDocumentMeter.isPresent()).isTrue();
         PrometheusTimer timer = (PrometheusTimer) ocrOnDocumentMeter.get();
         assertThat(timer.count()).isEqualTo(3);
@@ -85,7 +64,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
     @SneakyThrows
     public void testOcr() {
 
-        String text = testOCR("files/13485 cert example 2.pdf");
+        String text = testOCR("files/402Study.pdf");
     }
 
 
@@ -102,18 +81,18 @@ public class OcrServiceIntegrationTest extends AbstractTest {
         // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
         String text = testOCR("files/merge_images.pdf");
         assertThat(text).contains("Bodyweight change of dams with live young - group mean values",
-                                  "Control",
-                                  "mg/g day",
-                                  "10 mg/kg/day",
-                                  "20 mg/kg/",
-                                  "Days",
-                                  "50",
-                                  "-200",
-                                  "—250",
-                                  "150",
-                                  "200",
-                                  "250",
-                                  "—150");
+                "Control",
+                "mg/g day",
+                "10 mg/kg/day",
+                "20 mg/kg/",
+                "Days",
+                "50",
+                "-200",
+                "—250",
+                "150",
+                "200",
+                "250",
+                "—150");
     }
 
 
@@ -137,17 +116,18 @@ public class OcrServiceIntegrationTest extends AbstractTest {
     private String testOCR(String fileName) {
 
         ClassPathResource pdfFileResource = new ClassPathResource(fileName);
-        Path tmpDir = Path.of(OsUtils.getTemporaryDirectory()).resolve("OCR_TEST").resolve(Path.of(fileName).getFileName());
-        tmpDir.toFile().mkdirs();
-        var documentFile = tmpDir.resolve(Path.of("document.pdf"));
-        var viewerDocumentFile = tmpDir.resolve(Path.of("viewerDocument.pdf"));
-        Files.copy(pdfFileResource.getFile().toPath(), documentFile, StandardCopyOption.REPLACE_EXISTING);
-        Files.copy(pdfFileResource.getFile().toPath(), viewerDocumentFile, StandardCopyOption.REPLACE_EXISTING);
+        var originId = FileStorageService.getStorageId(TEST_DOSSIER_ID, "file", FileType.ORIGIN);
+        try (var fileStream = pdfFileResource.getInputStream()) {
+            storageService.storeObject(TenantContext.getTenantId(), originId, fileStream);
+        }
 
-        ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", false, tmpDir, documentFile.toFile(), viewerDocumentFile.toFile());
-        System.out.println("File:" + documentFile);
+        Path tmpFileName = Path.of(getTemporaryDirectory()).resolve(Path.of(fileName).getFileName());
+        try (var out = new FileOutputStream(tmpFileName.toFile())) {
+            ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", out);
+            System.out.println("File:" + tmpFileName);
+        }
 
-        try (var fileStream = new FileInputStream(documentFile.toFile())) {
+        try (var fileStream = new FileInputStream(tmpFileName.toFile())) {
             return extractAllTextFromDocument(fileStream);
         }
     }
@@ -157,7 +137,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
     @SneakyThrows
     public void testOcrForAllDMFiles() {
 
-        String dir = "/home/kschuettler/Dokumente/TestFiles/certificates/no-ocr";
+        String dir = "/home/kschuettler/Dokumente/TestFiles/syn-dm-testfiles/";
         List<File> foundFiles = Files.walk(Path.of(dir))
                 .sorted(Comparator.comparingLong(this::getFileSize))
                 .map(Path::toFile)
@@ -167,9 +147,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
         int fileCount = foundFiles.size();
         AtomicInteger processedCount = new AtomicInteger();
         System.out.printf("Found %s files, starting OCR for each.%n%n", fileCount);
-        foundFiles.stream()
-                .peek(file -> System.out.printf("%s/%s: %s%n", processedCount.getAndIncrement(), fileCount, file))
-                .forEach(this::testOCRForFile);
+        foundFiles.stream().peek(file -> System.out.printf("%s/%s: %s%n", processedCount.getAndIncrement(), fileCount, file)).forEach(this::testOCRForFile);
     }
 
 
@@ -188,161 +166,21 @@ public class OcrServiceIntegrationTest extends AbstractTest {
     }
 
 
+
     @SneakyThrows
     private void testOCRForFile(File file) {
 
-        Path tmpDir = Path.of(OsUtils.getTemporaryDirectory()).resolve("OCR_TEST").resolve(file.toPath().getFileName());
-        tmpDir.toFile().mkdirs();
-        var documentFile = tmpDir.resolve(Path.of("document.pdf"));
-        var viewerDocumentFile = tmpDir.resolve(Path.of("viewerDocument.pdf"));
-        Files.copy(file.toPath(), documentFile, StandardCopyOption.REPLACE_EXISTING);
-        Files.copy(file.toPath(), viewerDocumentFile, StandardCopyOption.REPLACE_EXISTING);
+        var originId = FileStorageService.getStorageId(TEST_DOSSIER_ID, "file", FileType.ORIGIN);
+        try (var fileStream = new FileInputStream(file)) {
+            storageService.storeObject(TenantContext.getTenantId(), originId, fileStream);
+        }
 
-        ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", false, tmpDir, documentFile.toFile(), viewerDocumentFile.toFile());
-        System.out.println("File:" + documentFile);
+        Path tmpFileName = Path.of(getTemporaryDirectory()).resolve(Path.of(file.getAbsolutePath()).getFileName());
+        try (var out = new FileOutputStream(tmpFileName.toFile())) {
+            ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", out);
+            System.out.println("File:" + tmpFileName);
+        }
         System.out.println("\n\n");
     }
 
-
-    @SneakyThrows
-    @Test
-    public void testMakeTextVisible() {
-
-        File file = new File("/home/kschuettler/Downloads/18-Curacron_ToxicidadeOcularInVitro.pdf");
-        PDFDoc doc;
-        try (var in = new FileInputStream(file)) {
-
-            doc = new PDFDoc(in);
-        }
-        execute(doc, false, false, Collections.emptySet());
-
-        try (var out = new FileOutputStream("/tmp/test.pdf")) {
-            doc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
-        }
-    }
-
-
-    @SneakyThrows
-    private void execute(PDFDoc pdfDoc, boolean delta, boolean removePaths, Set<String> markedContentToIgnore) {
-
-        ElementWriter writer = new ElementWriter();
-        ElementReader reader = new ElementReader();
-        Set<Long> visitedXObjIds = new TreeSet<>();
-
-        for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
-
-            Page page = iterator.next();
-
-            visitedXObjIds.add(page.getSDFObj().getObjNum());
-
-            Context context = Context.builder()
-                    .reader(reader)
-                    .clippingPathStack(new ClippingPathStack(page.getMediaBox()))
-                    .markedContentStack(new MarkedContentStack())
-                    .removePaths(removePaths)
-                    .delta(delta)
-                    .overlappedElements(new ArrayList<>())
-                    .visibleElements(new ArrayList<>())
-                    .visitedXObjIds(visitedXObjIds)
-                    .markedContentToIgnore(markedContentToIgnore)
-                    .build();
-
-            removeClippedElementsAndInvisibleTextAndRememberOverlappedElements(page, writer, context);
-
-        }
-        writer.destroy();
-        reader.destroy();
-
-    }
-
-
-    private void removeClippedElementsAndInvisibleTextAndRememberOverlappedElements(Page page, ElementWriter writer, Context context) throws PDFNetException {
-
-        context.reader().begin(page);
-        context.markedContentStack().clear();
-        writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
-        processElements(writer, context);
-        writer.end();
-        context.reader().end();
-    }
-
-
-    private void processElements(ElementWriter writer, Context context) throws PDFNetException {
-
-        for (Element element = context.reader().next(); element != null; element = context.reader().next()) {
-
-            if (context.markedContentStack().currentMarkedContentContainsAny(context.markedContentToIgnore()) && element.getType() != Element.e_marked_content_end) {
-                writer.writeElement(element);
-                continue;
-            }
-
-            switch (element.getType()) {
-                case Element.e_text -> processText(element, writer, context);
-                case Element.e_form -> processForm(element, writer, context);
-                default -> writer.writeElement(element);
-            }
-        }
-    }
-
-
-    private void processForm(Element formElement, ElementWriter writer, Context context) throws PDFNetException {
-
-        writer.writeElement(formElement);
-        Obj formObj = formElement.getXObject();
-
-        if (!context.visitedXObjIds().contains(formObj.getObjNum())) {
-            context.visitedXObjIds().add(formObj.getObjNum());
-            // writer needs to be newly initialized when entering a new content stream
-            // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
-            ElementWriter formWriter = new ElementWriter();
-            context.reader().formBegin();
-            formWriter.begin(formObj);
-
-            context.reader().clearChangeList();
-            formWriter.setDefaultGState(context.reader());
-
-            processElements(formWriter, context);
-            formWriter.end();
-            formWriter.destroy();
-            context.reader().end();
-        }
-    }
-
-
-    private void processText(Element textElement, ElementWriter writer, Context context) throws PDFNetException {
-
-        Rect textBBox = textElement.getBBox();
-
-        if (textBBox == null) {
-            writer.writeElement(textElement);
-            return;
-        }
-
-        GState gState = textElement.getGState();
-
-        gState.setFillColorSpace(ColorSpace.createDeviceRGB());
-        // blue for elements removed due to transparency or not rendered or same color as background
-        gState.setFillColor(new ColorPt(0, 0, 1));
-        gState.setTextRenderMode(GState.e_fill_text);
-        gState.setFillOpacity(1);
-        writer.writeElement(textElement);
-
-    }
-
-
-    @Builder
-    record Context(
-            boolean removePaths,
-            boolean delta,
-            ElementReader reader,
-            ClippingPathStack clippingPathStack,
-            MarkedContentStack markedContentStack,
-            List<ElementFeatures> overlappedElements,
-            List<ElementFeatures> visibleElements,
-            Set<Long> visitedXObjIds,
-            Set<String> markedContentToIgnore
-    ) {
-
-    }
-
 }

publish-custom-image.sh

@@ -11,5 +11,5 @@ commit_hash=$(git rev-parse --short=5 HEAD)
 # Combine branch and commit hash
 buildName="${USER}-${branch}-${commit_hash}"
 
-gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
+gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName --no-build-cache
 echo "nexus.knecon.com:5001/ff/${dir}-server:$buildName"