Merge branch 'RED-6686' into 'master'

RED-6686 Extract Tenant and user-management code into a separate service. See merge request redactmanager/ocr-service!1
RED-6686 Extract Tenant and user-management code into a separate service.
2023-06-26 21:58:50 +02:00 · 2023-06-26 21:58:49 +02:00 · 2023-06-21 16:15:17 +02:00 · 2023-06-21 16:11:07 +02:00 · 2023-06-15 14:08:39 +02:00 · 2023-06-15 14:37:32 +03:00
44 changed files with 968 additions and 754 deletions
@@ -0,0 +1,6 @@
+variables:
+  SONAR_PROJECT_KEY: 'LM_license-service'
+include:
+  - project: 'gitlab/gitlab'
+    ref: 'main'
+    file: 'ci-templates/maven_java.yml'
@@ -1,37 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.atlassian.bamboo</groupId>
-        <artifactId>bamboo-specs-parent</artifactId>
-        <version>8.1.3</version>
-        <relativePath/>
-    </parent>
-
-    <artifactId>bamboo-specs</artifactId>
-    <version>1.0.0-SNAPSHOT</version>
-    <packaging>jar</packaging>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.atlassian.bamboo</groupId>
-            <artifactId>bamboo-specs-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.atlassian.bamboo</groupId>
-            <artifactId>bamboo-specs</artifactId>
-        </dependency>
-
-        <!-- Test dependencies -->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-
-    <!-- run 'mvn test' to perform offline validation of the plan -->
-    <!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
-</project>
@@ -1,125 +0,0 @@
-package buildjob;
-
-import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
-
-import java.time.LocalTime;
-
-import com.atlassian.bamboo.specs.api.BambooSpec;
-import com.atlassian.bamboo.specs.api.builders.BambooKey;
-import com.atlassian.bamboo.specs.api.builders.Variable;
-import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
-import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
-import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
-import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions;
-import com.atlassian.bamboo.specs.api.builders.plan.Job;
-import com.atlassian.bamboo.specs.api.builders.plan.Plan;
-import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
-import com.atlassian.bamboo.specs.api.builders.plan.Stage;
-import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
-import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
-import com.atlassian.bamboo.specs.api.builders.project.Project;
-import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
-import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
-import com.atlassian.bamboo.specs.builders.task.ScriptTask;
-import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
-import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
-import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
-import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
-import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
-import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
-import com.atlassian.bamboo.specs.util.BambooServer;
-
-/**
- * Plan configuration for Bamboo.
- * Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
- */
-@BambooSpec
-public class PlanSpec {
-
-    private static final String SERVICE_NAME = "ocr-service";
-
-    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
-
-
-    /**
-     * Run main to publish plan on Bamboo
-     */
-    public static void main(final String[] args) throws Exception {
-        //By default credentials are read from the '.credentials' file.
-        BambooServer bambooServer = new BambooServer("http://localhost:8085");
-
-        Plan plan = new PlanSpec().createPlan();
-        bambooServer.publish(plan);
-        PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
-        bambooServer.publish(planPermission);
-
-        Plan secPlan = new PlanSpec().createSecBuild();
-        bambooServer.publish(secPlan);
-        PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
-        bambooServer.publish(secPlanPermission);
-    }
-
-
-    private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
-
-        Permissions permission = new Permissions().userPermissions("atlbamboo",
-                        PermissionType.EDIT,
-                        PermissionType.VIEW,
-                        PermissionType.ADMIN,
-                        PermissionType.CLONE,
-                        PermissionType.BUILD)
-                .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
-                .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
-                .loggedInUserPermissions(PermissionType.VIEW)
-                .anonymousUserPermissionView();
-        return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
-    }
-
-
-    private Project project() {
-
-        return new Project().name("RED").key(new BambooKey("RED"));
-    }
-
-
-    public Plan createPlan() {
-
-        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
-                .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                                        .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
-                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
-                                new ScriptTask().description("Build").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh").argument(SERVICE_NAME),
-                                createJUnitParserTask().description("Resultparser")
-                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                        .enabled(true),
-                                new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL),
-                                new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(new BitbucketServerTrigger())
-                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
-                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
-                        .notificationForCommitters());
-    }
-
-
-    public Plan createSecBuild() {
-
-        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                                        .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
-                                new VcsCheckoutTask().description("Checkout Default Repository").checkoutItems(new CheckoutItem().defaultRepository()),
-                                new ScriptTask().description("Sonar").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh").argument(SERVICE_NAME))
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
-                                .dockerRunArguments("--net=host")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
-                .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
-    }
-
-}
@@ -1,62 +0,0 @@
-#!/bin/bash
-set -e
-
-SERVICE_NAME=$1
-
-if [[ "$bamboo_planRepository_branchName" == "master" ]]
-then
-	echo "building on master branch"
-    branchVersion=$(cat pom.xml | grep -Eo "<version>.*" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|')
-	echo "branch version is : $branchVersion"
-    latestVersion=$(semver $( git tag -l $branchVersion.* ) | tail -n1)
-	echo "latestVersion is : $latestVersion"
-    newVersion="$(semver $latestVersion -p -i minor)"
-	echo "newVersion is : $newVersion"
-elif [[ "$bamboo_planRepository_branchName" == release* ]]
-then
-    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
-    latestVersion=$(semver $( git tag -l $branchVersion.* ) | tail -n1)
-    newVersion="$(semver $latestVersion -p -i patch)"
-elif [[ "${bamboo_version_tag}" != "dev" ]]
-then
-    newVersion="${bamboo_version_tag}"
-else
-    mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        --no-transfer-progress \
-        ${bamboo_maven_add_param} \
-        clean install \
-        -Djava.security.egd=file:/dev/./urandomelse
-    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
-    exit 0
-fi
-
-echo "gitTag=${newVersion}" > git.tag
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    versions:set \
-    -DnewVersion=${newVersion}
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    versions:set  \
-    -DnewVersion=${newVersion}
-
-mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    --no-transfer-progress \
-    clean deploy \
-    ${bamboo_maven_add_param} \
-    -e \
-    -DdeployAtEnd=true \
-    -Dmaven.wagon.http.ssl.insecure=true \
-    -Dmaven.wagon.http.ssl.allowall=true \
-    -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    package
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    docker:push
@@ -1,44 +0,0 @@
-#!/bin/bash
-set -e
-
-SERVICE_NAME=$1
-
-echo "build jar binaries"
-mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    --no-transfer-progress \
-    clean install \
-    -Djava.security.egd=file:/dev/./urandomelse
-
-echo "dependency-check:aggregate"
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    org.owasp:dependency-check-maven:aggregate
-
-if [[ -z "${bamboo_repository_pr_key}" ]]
-then
-    echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
-    mvn --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        sonar:sonar \
-        -Dsonar.projectKey=RED_$SERVICE_NAME \
-        -Dsonar.host.url=https://sonarqube.iqser.com \
-        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-        -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
-        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-else
-    echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
-    mvn --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        sonar:sonar \
-        -Dsonar.projectKey=RED_$SERVICE_NAME \
-        -Dsonar.host.url=https://sonarqube.iqser.com \
-        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-        -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
-        -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
-        -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
-        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-fi
@@ -1,22 +0,0 @@
-package buildjob;
-
-import org.junit.Test;
-
-import com.atlassian.bamboo.specs.api.builders.plan.Plan;
-import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException;
-import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
-
-public class PlanSpecTest {
-
-    @Test
-    public void checkYourPlanOffline() throws PropertiesValidationException {
-
-        Plan plan = new PlanSpec().createPlan();
-        EntityPropertiesBuilders.build(plan);
-
-        Plan secPlan = new PlanSpec().createSecBuild();
-        EntityPropertiesBuilders.build(secPlan);
-
-    }
-
-}
@@ -3,16 +3,16 @@
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
-        <groupId>com.iqser.red</groupId>
+        <groupId>com.knecon.fforesight</groupId>
        <artifactId>platform-docker-dependency</artifactId>
-        <version>1.2.0</version>
+        <version>0.1.0</version>
        <relativePath/>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>ocr-service-image-v1</artifactId>
    <groupId>com.iqser.red.service</groupId>
-    <version>1.0-SNAPSHOT</version>
+    <version>3.0-SNAPSHOT</version>
    <packaging>pom</packaging>

    <properties>
@@ -1,10 +1,14 @@
-FROM red/base-image:2.0.0
+FROM red/base-image:2.0.2

 COPY "libs/pdftron/OCRModuleLinux.tar.gz" .
 RUN tar xvzf OCRModuleLinux.tar.gz
 RUN mkdir /OCRModule
 RUN mv Lib/* /OCRModule/

+RUN apt-get -y update
+# Ghostscript somehow improves ocr quality using pdfton, do not remove!
+RUN apt-get -y install ghostscript
+
 ARG PLATFORM_JAR

 ENV PLATFORM_JAR ${PLATFORM_JAR}
@@ -7,27 +7,17 @@
    <parent>
        <groupId>com.iqser.red.service</groupId>
        <artifactId>ocr-service-v1</artifactId>
-        <version>1.0-SNAPSHOT</version>
+        <version>3.0-SNAPSHOT</version>
    </parent>

    <artifactId>ocr-service-api-v1</artifactId>
-    <version>1.0-SNAPSHOT</version>

    <properties>
-        <persistence-service.version>1.269.0</persistence-service.version>
-        <redaction-service.version>3.155.0</redaction-service.version>
-        <dsljson.version>1.9.9</dsljson.version>
+        <persistence-service.version>2.70.0</persistence-service.version>
    </properties>

    <dependencies>

-        <!-- https://mvnrepository.com/artifact/com.dslplatform/dsl-json-java8 -->
-        <dependency>
-            <groupId>com.dslplatform</groupId>
-            <artifactId>dsl-json-java8</artifactId>
-            <version>${dsljson.version}</version>
-        </dependency>
-
        <dependency>
            <!-- This dependency contains annotations that are used in specifying REST endpoints. -->
            <!-- It is optional since not all users of this API might use Feign. -->
@@ -43,26 +33,9 @@

        <dependency>
            <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-api-v1</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.iqser.red.service</groupId>
-                    <artifactId>redaction-service-api-v1</artifactId>
-                </exclusion>
-            </exclusions>
+            <artifactId>persistence-service-internal-api-v1</artifactId>
            <version>${persistence-service.version}</version>
        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>redaction-service-api-v1</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.iqser.red.service</groupId>
-                    <artifactId>persistence-service-api-v1</artifactId>
-                </exclusion>
-            </exclusions>
-            <version>${redaction-service.version}</version>
-        </dependency>

    </dependencies>
 </project>
@@ -0,0 +1,17 @@
+package com.iqser.red.service.ocr.v1.api.model;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+public class IdentityTest {
+
+    @Test
+    public void mockTest() {
+
+        int i = 1;
+        assertThat(i).isEqualTo(1);
+    }
+
+}
+
@@ -7,23 +7,35 @@
    <parent>
        <groupId>com.iqser.red.service</groupId>
        <artifactId>ocr-service-v1</artifactId>
-        <version>1.0-SNAPSHOT</version>
+        <version>3.0-SNAPSHOT</version>
    </parent>

    <artifactId>ocr-service-server-v1</artifactId>
-    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <tennat-commons.version>0.6.0</tennat-commons.version>
+    </properties>

    <dependencies>
+        <dependency>
+            <groupId>com.knecon.fforesight</groupId>
+            <artifactId>tenant-commons</artifactId>
+            <version>${tennat-commons.version}</version>
+        </dependency>
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>ocr-service-api-v1</artifactId>
            <version>${project.version}</version>
        </dependency>
-
        <dependency>
            <groupId>com.iqser.red.commons</groupId>
            <artifactId>storage-commons</artifactId>
        </dependency>
+        <dependency>
+            <groupId>com.iqser.red.commons</groupId>
+            <artifactId>pdftron-logic-commons</artifactId>
+            <version>2.3.0</version>
+        </dependency>
        <dependency>
            <groupId>com.iqser.red.commons</groupId>
            <artifactId>spring-commons</artifactId>
@@ -39,17 +51,16 @@
        <dependency>
            <groupId>com.pdftron</groupId>
            <artifactId>PDFNet</artifactId>
-            <version>9.4.0</version>
+            <version>10.1.0</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-amqp</artifactId>
-            <version>2.3.1.RELEASE</version>
        </dependency>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-kms</artifactId>
-            <version>1.12.158</version>
+            <version>1.12.440</version>
        </dependency>
        <dependency>
            <groupId>com.google.guava</groupId>
@@ -65,7 +76,6 @@
        <dependency>
            <groupId>org.springframework.amqp</groupId>
            <artifactId>spring-rabbit-test</artifactId>
-            <version>2.3.1</version>
            <scope>test</scope>
        </dependency>
        <dependency>
@@ -81,21 +91,10 @@
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
-            <version>3.12.0</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <configuration>
-                    <annotationProcessors>
-                        <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
-                        <annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
-                    </annotationProcessors>
-                </configuration>
-            </plugin>
            <plugin>
                <!-- generate git.properties for exposure in /info -->
                <groupId>pl.project13.maven</groupId>
@@ -114,6 +113,15 @@
                    </execution>
                </executions>
            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <annotationProcessors>
+                        <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
+                    </annotationProcessors>
+                </configuration>
+            </plugin>
            <plugin>
                <!-- repackages the generated jar into a runnable fat-jar and makes it
                    executable -->
@@ -2,6 +2,7 @@ package com.iqser.red.service.ocr.v1.server;

 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
+import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -10,18 +11,21 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.annotation.EnableAsync;

-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
 import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;

@EnableAsync
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@EnableConfigurationProperties(OcrServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
-@Import({DefaultWebMvcConfiguration.class, MessagingConfiguration.class})
+@Import({ MessagingConfiguration.class,  StorageAutoConfiguration.class})
@EnableFeignClients(basePackageClasses = FileStatusProcessingUpdateClient.class)
 public class Application {

@@ -42,4 +46,11 @@ public class Application {
        return new TimedAspect(registry);
    }

+
+    @Bean
+    public InvisibleElementRemovalService invisibleElementRemovalService() {
+
+        return new InvisibleElementRemovalService();
+    }
+
 }
@@ -2,7 +2,7 @@ package com.iqser.red.service.ocr.v1.server.client;

 import org.springframework.cloud.openfeign.FeignClient;

-import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;

@FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
 public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {
@@ -20,6 +20,9 @@ public class MessagingConfiguration {

    public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue";

+    public static final String X_ERROR_INFO_HEADER = "x-error-message";
+    public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
+

    @Bean
    public Queue ocrQueue() {
@@ -1,14 +1,12 @@
 package com.iqser.red.service.ocr.v1.server.initializer;

-import javax.annotation.PostConstruct;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
 import com.pdftron.pdf.PDFNet;

+import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;

@Component
@RequiredArgsConstructor
@@ -29,7 +27,5 @@ public class PDFNetInitializer {
        PDFNet.setTempPath("/tmp/pdftron");
        PDFNet.addResourceSearchPath(ocrModulePath);
        PDFNet.initialize(pdftronLicense);
-
    }
-
 }
@@ -1,6 +1,7 @@
 package com.iqser.red.service.ocr.v1.server.model;

-import com.iqser.red.service.redaction.v1.model.Rectangle;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;

 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -3,15 +3,12 @@ package com.iqser.red.service.ocr.v1.server.model.image;
 import java.util.ArrayList;
 import java.util.List;

-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;

 import lombok.Data;

@Data
-@CompiledJson
 public class ImageServiceResponse {

    private String dossierId;
@@ -19,13 +16,11 @@ public class ImageServiceResponse {

    @JsonProperty(value = "imageMetadata")
    @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
    private List<ImageMetadata> data = new ArrayList<>();


    @JsonProperty(value = "imageMetadata")
    @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
    public void setData(List<ImageMetadata> data) {this.data = data;}

 }
@@ -7,8 +7,9 @@ import org.apache.commons.io.IOUtils;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.ocr.v1.server.model.image.ImageServiceResponse;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.tenantcommons.TenantContext;

 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -31,39 +32,39 @@ public class FileStorageService {
    @SneakyThrows
    public byte[] getOriginalFile(String dossierId, String fileId) {

-        return IOUtils.toByteArray(storageService.getObject(getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
+        return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
    }


    @SneakyThrows
    public InputStream getOriginalFileAsStream(String dossierId, String fileId) {

-        return storageService.getObject(getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
+        return storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
    }


    public void storeOriginalFile(String dossierId, String fileId, InputStream stream) {

-        storageService.storeObject(getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
+        storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
    }


    public boolean untouchedFileExists(String dossierId, String fileId) {

-        return storageService.objectExists(getStorageId(dossierId, fileId, FileType.UNTOUCHED));
+        return storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED));
    }


    public void storeUntouchedFile(String dossierId, String fileId, byte[] data) {

-        storageService.storeObject(getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
+        storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
    }


    @SneakyThrows
    public ImageServiceResponse getImageServiceResponse(String dossierId, String fileId) {

-        return storageService.readJSONObject(getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
+        return storageService.readJSONObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
    }

 }
@@ -0,0 +1,196 @@
+package com.iqser.red.service.ocr.v1.server.service;
+
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.pdftron.common.Matrix2D;
+import com.pdftron.common.PDFNetException;
+import com.pdftron.pdf.Element;
+import com.pdftron.pdf.ElementReader;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.pdf.Rect;
+import com.pdftron.pdf.RectCollection;
+
+import lombok.SneakyThrows;
+
+@Service
+public class ImagePositionRetrievalService {
+
+    private static final double TOLERANCE = 1e-1;
+
+    // any image with smaller height and width than this gets thrown out, see everyPointInDashedLineIsImage.pdf
+    private static final int PIXEL_THRESHOLD = 10;
+
+
+    /**
+     * Iterates over all elements in a PDF Document and retrieves the bounding box for each image, that is larger than the pixel threshold of 10 in either dimension.
+     * Then it adjusts the bounding boxes for the page rotation.
+     * If the mirrorY flag is set, the Y Coordinates are mirrored and moved up by the page height. This is required for PDFTrons OCRModule.
+     *
+     * @param pdfDoc  a PDF File as PDFTron PDFDoc class
+     * @param mirrorY if this flag is set, all coordinates are calculated with upper left corner as (0,0), else initial user space
+     * @return a map with the page indices as keys and the image bounding boxes on that page as a RectCollection
+     */
+    @SneakyThrows
+    public Map<Integer, RectCollection> getImagePositionPerPage(PDFDoc pdfDoc, boolean mirrorY) {
+
+        Map<Integer, RectCollection> pageIdToImagePositions = new HashMap<>();
+        ElementReader reader = new ElementReader();
+        for (int pageId = 1; pageId <= pdfDoc.getPageCount(); ++pageId) {
+            RectCollection imagePositions = new RectCollection();
+
+            reader.begin(pdfDoc.getPage(pageId));
+            findImagePositionsOnPage(reader, imagePositions, pdfDoc.getPage(pageId), mirrorY);
+            imagePositions = mergeOverlappingRects(imagePositions);
+            reader.end();
+
+            if (imagePositions.getNumRects() > 0) {
+                pageIdToImagePositions.put(pageId, imagePositions);
+            }
+        }
+        reader.destroy();
+        return pageIdToImagePositions;
+    }
+
+
+    private void findImagePositionsOnPage(ElementReader reader, RectCollection imagePositions, Page currentPage, boolean mirrorY) throws PDFNetException {
+
+        Element element;
+        while ((element = reader.next()) != null) {
+            switch (element.getType()) {
+                case Element.e_image, Element.e_inline_image -> {
+                    // see everyPointInDashedLineIsImage.pdf TestFile
+                    if (element.getImageHeight() > PIXEL_THRESHOLD || element.getImageWidth() > PIXEL_THRESHOLD) {
+                        imagePositions.addRect(toRotationAdjustedRect(element.getBBox(), currentPage, mirrorY));
+                    }
+                }
+                case Element.e_form -> {
+                    reader.formBegin();
+                    findImagePositionsOnPage(reader, imagePositions, currentPage, mirrorY);
+                    reader.end();
+                }
+            }
+        }
+    }
+
+
+    @SneakyThrows
+    public RectCollection mergeOverlappingRects(RectCollection imagePositions) {
+
+        if (imagePositions.getNumRects() < 2) {
+            return imagePositions;
+        }
+
+        List<Rectangle2D> rectangleList = toSortedRectangleList(imagePositions);
+
+        mergeRectangleList(rectangleList);
+
+        return toRectCollection(rectangleList);
+    }
+
+
+    // Sometimes images are split up into stripes, here we merge the positions of aligned and intersecting rectangles into one larger rectangle
+    private void mergeRectangleList(List<Rectangle2D> rectangleList) {
+
+        for (int idx = 0; rectangleList.size() >= idx + 2; ) {
+
+            var rect1 = rectangleList.get(idx);
+            var rect2 = rectangleList.get(idx + 1);
+
+            if (intersects(rect1, rect2) && isAlignedXOrY(rect1, rect2)) {
+                rectangleList.remove(idx + 1);
+                rectangleList.remove(idx);
+                rectangleList.add(idx, rect1.createUnion(rect2));
+            } else {
+                ++idx;
+            }
+        }
+    }
+
+
+    private boolean intersects(Rectangle2D rect1, Rectangle2D rect2) {
+
+        return rect1.intersects(rect2.getMinX() - TOLERANCE, rect2.getMinY() - TOLERANCE, rect2.getWidth() + (2 * TOLERANCE), rect2.getHeight() + (2 * TOLERANCE));
+    }
+
+
+    private boolean isAlignedXOrY(Rectangle2D rect1, Rectangle2D rect2) {
+
+        boolean isAlignedX = Math.abs(rect1.getMinX() - rect2.getMinX()) < TOLERANCE && Math.abs(rect1.getMaxX() - rect2.getMaxX()) < TOLERANCE;
+        boolean isAlignedY = Math.abs(rect1.getMinY() - rect2.getMinY()) < TOLERANCE && Math.abs(rect1.getMaxY() - rect2.getMaxY()) < TOLERANCE;
+
+        return isAlignedX || isAlignedY;
+    }
+
+
+    private Rect toRotationAdjustedRect(Rect bbox, Page page, boolean mirrorY) throws PDFNetException {
+
+        int rotation = page.getRotation();
+        double height = page.getPageHeight();
+        double width = page.getPageWidth();
+
+        // Even though PDFTron almost always has the origin in the lower left corner, for some reason, the OCRModule's addTextZonesForPage() uses the upper left corner as origin...
+        Matrix2D mirrorMatrix;
+        if (mirrorY) {
+            mirrorMatrix = new Matrix2D(1, 0, 0, -1, 0, height);
+        } else {
+            mirrorMatrix = new Matrix2D();
+        }
+
+        // We need to rotate the rects to fit to the page rotation
+        Matrix2D rotationMatrix = switch (rotation) {
+            case 1 -> new Matrix2D(0, -1, 1, 0, 0, height);
+            case 2 -> new Matrix2D(-1, 0, 0, -1, width, height);
+            case 3 -> new Matrix2D(0, 1, -1, 0, width, 0);
+            default -> new Matrix2D();
+        };
+
+        Matrix2D finalMatrix = mirrorMatrix.multiply(rotationMatrix);
+
+        Point2D.Double p1 = finalMatrix.multPoint(bbox.getX1(), bbox.getY1());
+        Point2D.Double p2 = finalMatrix.multPoint(bbox.getX2(), bbox.getY2());
+
+        // PDFTron Rect *needs* lower left and upper right coordinates to calculate width and height correctly, even though the documentation states otherwise
+        Point2D.Double lowerLeft = new Point2D.Double(Math.min(p1.x, p2.x), Math.min(p1.y, p2.y));
+        Point2D.Double upperRight = new Point2D.Double(Math.max(p1.x, p2.x), Math.max(p1.y, p2.y));
+
+        return new Rect(lowerLeft.x, lowerLeft.y, upperRight.x, upperRight.y);
+    }
+
+
+    private RectCollection toRectCollection(List<Rectangle2D> rectangleList) {
+
+        RectCollection rectCollection = new RectCollection();
+        rectangleList.forEach(r -> {
+            try {
+                rectCollection.addRect(new Rect(r.getMinX(), r.getMinY(), r.getMaxX(), r.getMaxY()));
+            } catch (PDFNetException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        return rectCollection;
+    }
+
+
+    @SneakyThrows
+    private List<Rectangle2D> toSortedRectangleList(RectCollection rectCollection) {
+
+        List<Rectangle2D> list = new LinkedList<>();
+        for (int i = 0; i < rectCollection.getNumRects(); ++i) {
+            Rect r = rectCollection.getRectAt(i);
+            list.add(new Rectangle2D.Double(r.getX1(), r.getY1(), r.getWidth(), r.getHeight()));
+        }
+        list.sort(Comparator.comparingDouble(RectangularShape::getMinY).thenComparing(RectangularShape::getMinX));
+        return list;
+    }
+
+}
@@ -1,43 +1,30 @@
 package com.iqser.red.service.ocr.v1.server.service;

+import static java.lang.String.format;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collections;
+import java.io.OutputStream;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;

-import org.apache.commons.io.IOUtils;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
 import com.iqser.red.service.ocr.v1.api.model.OCRStatusUpdateResponse;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
-import com.iqser.red.service.ocr.v1.server.model.ImagePosition;
-import com.iqser.red.service.ocr.v1.server.model.image.ImageServiceResponse;
 import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
-import com.iqser.red.service.persistence.service.v1.api.utils.SuppressFBWarnings;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.pdftron.common.PDFNetException;
-import com.pdftron.pdf.Element;
-import com.pdftron.pdf.ElementReader;
-import com.pdftron.pdf.ElementWriter;
 import com.pdftron.pdf.OCRModule;
 import com.pdftron.pdf.OCROptions;
 import com.pdftron.pdf.Optimizer;
 import com.pdftron.pdf.PDFDoc;
 import com.pdftron.pdf.Page;
-import com.pdftron.pdf.PageIterator;
-import com.pdftron.pdf.Rect;
 import com.pdftron.pdf.RectCollection;
-import com.pdftron.sdf.Obj;
 import com.pdftron.sdf.SDFDoc;

 import io.micrometer.core.annotation.Timed;
@@ -59,267 +46,137 @@ public class OCRService {

    private final ObjectMapper objectMapper;

+    private final InvisibleElementRemovalService invisibleElementRemovalService;

-    @Timed("redactmanager_PDFTron-ocrDocument")
-    @SneakyThrows
-    public InputStream ocrDocument(String dossierId, String fileId) {
-
-        var fileStream = fileStorageService.getOriginalFileAsStream(dossierId, fileId);
-
-        var imageServiceResponse = fileStorageService.getImageServiceResponse(dossierId, fileId);
-
-        var fileBytes = IOUtils.toByteArray(fileStream);
-
-        var ocrBytes = ocr(fileBytes, fileId, imageServiceResponse);
-
-        return new ByteArrayInputStream(ocrBytes);
-
-    }
-
-
-    @SuppressFBWarnings("REC_CATCH_EXCEPTION")
-    private byte[] ocr(byte[] file, String fileId, ImageServiceResponse imageServiceResponse) {
-
-        PDFDoc pdfDoc = null;
-        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
-            pdfDoc = new PDFDoc(file);
-            removeInvisibleText(pdfDoc);
-
-            Map<Integer, List<ImagePosition>> pages = new HashMap<>();
-
-            // TODO take logic to ignore small and combine images from image-service.
-            // TODO Then replace logic so ocr-service is independent from image-service.
-            imageServiceResponse.getData()
-                    .forEach(imageMetadata -> pages.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
-                            .add(new ImagePosition(new Rectangle(new Point(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1()),
-                                    imageMetadata.getGeometry().getWidth(),
-                                    imageMetadata.getGeometry().getHeight(),
-                                    imageMetadata.getPosition().getPageNumber()), imageMetadata.isAlpha())));
-
-            Map<Integer, PDFDoc> pdfDocMap = Collections.synchronizedMap(new HashMap<>());
-
-            rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
-                    objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pages.keySet().size()).build()));
-
-            ocrPages(pdfDoc, fileId, pages, pdfDocMap);
-
-            for (var entry : pdfDocMap.entrySet()) {
-
-                var ocrDoc = entry.getValue();
-                var page = entry.getKey();
-
-                Page ocrPage = ocrDoc.getPageIterator(1).next();
-                pdfDoc.pageInsert(pdfDoc.getPageIterator(page), ocrPage);
-                pdfDoc.pageRemove(pdfDoc.getPageIterator(page + 1));
-
-                ocrDoc.close();
-            }
-
-            Optimizer.optimize(pdfDoc);
-            pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
-            pdfDoc.close();
-
-            rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
-                    objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder()
-                            .fileId(fileId)
-                            .numberOfPagesToOCR(pages.keySet().size())
-                            .numberOfOCRedPages(pages.keySet().size())
-                            .ocrFinished(true)
-                            .build()));
-
-            return out.toByteArray();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        } finally {
-            if (pdfDoc != null) {
-                try {
-                    pdfDoc.close();
-                } catch (Exception e) {
-                    log.debug("Failed to close document", e);
-                }
-            }
-        }
-    }
-
-
-    @SneakyThrows
-    private void ocrPages(PDFDoc pdfDoc, String fileId, Map<Integer, List<ImagePosition>> pages, Map<Integer, PDFDoc> pdfDocMap) {
-
-        int numberOfOCRedPages = 0;
-        for (var pageEntry : pages.entrySet()) {
-
-            try {
-                RectCollection rectCollection = new RectCollection();
-
-                var page = pageEntry.getKey();
-
-                Page pdfPage = pdfDoc.getPageIterator(page).next();
-
-                pdfPage.setMediaBox(pdfPage.getCropBox());
-
-                for (ImagePosition imagePosition : pageEntry.getValue()) {
-                    Rectangle rectangle = imagePosition.getRectangle();
-
-                    // Warning coordinate system is different in this call macOs/Linux
-                    double y = -rectangle.getTopLeft().getY() + pdfPage.getCropBox().getY2() - rectangle.getHeight();
-                    rectCollection.addRect(rectangle.getTopLeft().getX(), y, rectangle.getTopLeft().getX() + rectangle.getWidth(), y + rectangle.getHeight());
-                }
-
-                PDFDoc ocrDoc = new PDFDoc();
-                ocrDoc.pagePushBack(pdfPage);
-                pdfDocMap.put(pageEntry.getKey(), ocrDoc);
-
-                OCROptions options = new OCROptions();
-                options.addTextZonesForPage(rectCollection, 1);
-                options.addLang(ENGLISH);
-                options.addDPI(settings.getOcrDPI());
-                OCRModule.processPDF(ocrDoc, options);
-
-                rectCollection.clear();
-
-            } catch (Exception e) {
-                log.warn("Failed to process PDF page {} - {}", pageEntry.getKey(), e);
-            }
-
-            rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
-                    objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder()
-                            .fileId(fileId)
-                            .numberOfPagesToOCR(pages.keySet().size())
-                            .numberOfOCRedPages(++numberOfOCRedPages)
-                            .build()));
-
-            log.warn("Done page {}", pageEntry);
-
-        }
-    }
+    private final ImagePositionRetrievalService imagePositionRetrievalService;


    /**
-     * There are 2 possibilities to have invisible Text in pdfs.
-     * 1. gState is set to invisible, this is ocr text.
-     * 2. Filled Path elements in front of the text.
+     * First loads the PDF Document from storage.
+     * Then removes all invisible Elements from the PDF, check InvisibleElementRemovalService for details.
+     * Then gets Image Position Information, check ImagePositionRetrievalService for details.
+     * Then runs OCR page by page, exclusively on pages which have images on them. It does so, by creating a new PDFDoc and inserting a single page at a time.
+     * This is because PDFTron OCROptions overlays all regions where OCR should not be run with white images. It does not check for empty pages.
+     * For Documents with many pages but few Images this results in major performance improvements.
+     * It then re-adds the OCRed Pages to the original document and saves it.
+     *
+     * @param dossierId The dossier id
+     * @param fileId    The file id
+     * @param out       OutputStream to write the file to
     */
-    @SneakyThrows
-    private void removeInvisibleText(PDFDoc pdfDoc) {
+    @Timed("redactmanager_runOcrOnDocument")
+    public void runOcrOnDocument(String dossierId, String fileId, OutputStream out) throws IOException {

-        ElementWriter writer = new ElementWriter();
-        ElementReader reader = new ElementReader();
-        Set<Integer> visited = new TreeSet<>();
-
-        for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
-            Page page = iterator.next();
-            removeOverlapText(page, reader, writer, visited);
-        }
-    }
-
-
-    @SneakyThrows
-    private void removeOverlapText(Page page, ElementReader reader, ElementWriter writer, Set<Integer> visited) {
-
-        visited.add((int) page.getSDFObj().getObjNum());
-        reader.begin(page);
-        writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
-        processElements(reader, writer, visited, false);
-        writer.end();
-        reader.end();
-    }
-
-
-    @SneakyThrows
-    private void processElements(ElementReader reader, ElementWriter writer, Set<Integer> visited, boolean isInForm) {
-
-        Set<Rect> filledRectangles = new HashSet<>();
-        for (Element element = reader.next(); element != null; element = reader.next())
-
-            switch (element.getType()) {
-                case Element.e_image:
-                case Element.e_inline_image:
-                    processImage(element, writer, isInForm);
-                    break;
-
-                case Element.e_text:
-                    processText(element, writer, filledRectangles);
-                    break;
-
-                case Element.e_path:
-                    processPath(element, writer, filledRectangles);
-                    break;
-
-                case Element.e_form:
-                    processForm(reader, writer, element, visited);
-                    break;
-
-                default:
-                    writer.writeElement(element);
+        try (ByteArrayOutputStream transferOutputStream = new ByteArrayOutputStream()) {
+            try (InputStream fileStream = fileStorageService.getOriginalFileAsStream(dossierId, fileId)) {
+                invisibleElementRemovalService.removeInvisibleElements(fileStream, transferOutputStream, false);
+            }
+            try (InputStream transferInputStream = new ByteArrayInputStream(transferOutputStream.toByteArray())) {
+                long ocrStart = System.currentTimeMillis();
+                runOcr(transferInputStream, out, fileId);
+                long ocrEnd = System.currentTimeMillis();
+                log.info("ocr successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, format("%.1f", (ocrEnd - ocrStart) / 1000.0));
            }
-    }
-
-
-    @SneakyThrows
-    private void processImage(Element element, ElementWriter writer, boolean isInForm) {
-
-        if (!isInForm || !settings.isRemoveWatermark()) {
-            writer.writeElement(element);
        }
    }


    @SneakyThrows
-    private void processText(Element element, ElementWriter writer, Set<Rect> filledRectangles) {
+    private void runOcr(InputStream fileStream, OutputStream out, String fileId) {

-        if (element.getBBox() == null) {
-            writer.writeElement(element);
-            return;
-        }
+        PDFDoc pdfDoc = new PDFDoc(fileStream);

-        double x = element.getBBox().getX1();
-        double y = element.getBBox().getY1();
-        boolean filledRectangleIntersection = filledRectangles.stream().anyMatch(r -> {
+        Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, true);
+        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
+                OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).build());
+
+        // Optimization:
+        // When a page does not have a TextZone, PDFTron whites out the page. But, PDFTron scans it anyway, resulting in a longer runtime.
+        // So, we need to remove pages without images.
+        // Furthermore, creating a new document is *much* faster than reusing the same document and adding/removing pages one by one.
+        // Therefore, we create a new Document with a single page for every page that contains an image.
+        // For some reason, if we insert the OCR singlePageDoc into the original PDFDoc inside the loop, for some documents, the RAM Usage increases exponentially with every page.
+        // This is why, we replace the OCRed Pages outside the main loop.
+        int numProcessedPages = 0;
+        Map<Integer, PDFDoc> pageIdToSinglePagePdfDoc = new HashMap<>();
+        for (Integer pageId : pageIdToRectCollection.keySet()) {
            try {
-                return r.contains(x, y);
+                PDFDoc singlePagePdfDoc = extractSinglePagePdfDoc(pdfDoc, pageId);
+                processOcr(pageIdToRectCollection, pageId, singlePagePdfDoc);
+
+                log.info("{}/{} Page {} done, OCR regions {}",
+                        numProcessedPages,
+                        pageIdToRectCollection.size(),
+                        pageId,
+                        getAllOcrTextZonesAsString(pageIdToRectCollection, pageId));
+
+                pageIdToSinglePagePdfDoc.put(pageId, singlePagePdfDoc);
+                ++numProcessedPages;
+
+                rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
+                        OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).numberOfOCRedPages(numProcessedPages).build());
+
            } catch (PDFNetException e) {
-                throw new RuntimeException("Internal pdftron error during removal of overlap text", e);
+                log.error("Failed to process page {}", pageId);
+                throw new RuntimeException(e);
            }
-        });
-
-        var gState = element.getGState();
-
-        //See PDF Reference 5.3 Text rendering modes, 3 = Invisible, however this ocr does not use it.
-        if (!filledRectangleIntersection && gState.getTextRenderMode() != 3) {
-            writer.writeElement(element);
        }
+        log.info("Copying {} OCRed Pages into original Document", pageIdToSinglePagePdfDoc.size());
+        pageIdToSinglePagePdfDoc.forEach((pageId, singlePagePdfDoc) -> replaceOriginalPageWithOcrPage(pdfDoc, pageId, singlePagePdfDoc));
+        Optimizer.optimize(pdfDoc);
+
+        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
+                OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).numberOfOCRedPages(numProcessedPages).ocrFinished(true).build());
+
+        try {
+            pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
+        } catch (Exception e) {
+            log.error("Processed File with fileId {} could not be saved", fileId);
+            throw new RuntimeException(e);
+        }
+        pdfDoc.close();
+    }
+
+
+    private void processOcr(Map<Integer, RectCollection> pageIdToRectCollection, Integer pageId, PDFDoc singlePagePdfDoc) throws PDFNetException {
+
+        OCROptions options = new OCROptions();
+        options.addTextZonesForPage(pageIdToRectCollection.get(pageId), 1);
+        options.addLang(ENGLISH);
+        options.addDPI(settings.getOcrDPI());
+
+        OCRModule.processPDF(singlePagePdfDoc, options);
+    }
+
+
+    private static PDFDoc extractSinglePagePdfDoc(PDFDoc pdfDoc, Integer pageId) throws PDFNetException {
+
+        PDFDoc singlePagePdfDoc = new PDFDoc();
+        Page page = pdfDoc.getPage(pageId);
+        page.setMediaBox(page.getCropBox()); // this line ensures the ocr text is placed correctly by PDFTron, see TestFile MediaBoxBiggerThanCropBox.pdf
+        singlePagePdfDoc.pagePushBack(page);
+        return singlePagePdfDoc;
    }


    @SneakyThrows
-    private void processPath(Element element, ElementWriter writer, Set<Rect> filledRectangles) {
+    private static void replaceOriginalPageWithOcrPage(PDFDoc pdfDoc, Integer pageId, PDFDoc singlePagePdfDoc) {

-        if (element.getPathData() != null && element.getPathData().getPoints().length > 4) {
-            filledRectangles.add(element.getBBox());
-        }
-        writer.writeElement(element);
+        Page ocrPage = singlePagePdfDoc.getPage(1);
+        pdfDoc.pageInsert(pdfDoc.getPageIterator(pageId), ocrPage);
+        pdfDoc.pageRemove(pdfDoc.getPageIterator(pageId + 1));
+        singlePagePdfDoc.close();
    }


-    @SneakyThrows
-    private void processForm(ElementReader reader, ElementWriter writer, Element element, Set<Integer> visited) {
+    private static StringBuilder getAllOcrTextZonesAsString(Map<Integer, RectCollection> pageIdToRectCollection, Integer pageId) throws PDFNetException {

-        writer.writeElement(element);
-        Obj formObj = element.getXObject();
-
-        if (!visited.contains((int) formObj.getObjNum())) {
-            visited.add((int) formObj.getObjNum());
-            ElementWriter new_writer = new ElementWriter();
-            reader.formBegin();
-            new_writer.begin(formObj);
-
-            reader.clearChangeList();
-            new_writer.setDefaultGState(reader);
-
-            processElements(reader, new_writer, visited, true);
-            new_writer.end();
-            reader.end();
+        StringBuilder zonesString = new StringBuilder();
+        for (int j = 0; j < pageIdToRectCollection.get(pageId).getNumRects(); ++j) {
+            var r = pageIdToRectCollection.get(pageId).getRectAt(j);
+            zonesString.append(format("[lower left (%.1f|%.1f) upper right (%.1f|%.1f)]", r.getX1(), r.getY1(), r.getX2(), r.getY2()));
        }
+        return zonesString;
    }

 }
@@ -1,16 +1,26 @@
 package com.iqser.red.service.ocr.v1.server.service;

+import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
+import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
+
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
+import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Service;

-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.ocr.v1.api.model.DocumentRequest;
 import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
-import com.iqser.red.service.ocr.v1.api.model.DocumentRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;

 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
@@ -21,47 +31,60 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class OcrMessageReceiver {

-    private final ObjectMapper objectMapper;
    private final FileStorageService fileStorageService;
+    private final ObjectMapper objectMapper;
    private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
-
    private final OCRService ocrService;


    @RabbitHandler
    @RabbitListener(queues = MessagingConfiguration.OCR_QUEUE, concurrency = "1")
-    public void receiveOcr(String in) throws JsonProcessingException {
+    public void receiveOcr(Message in) throws IOException {

-        DocumentRequest ocrRequestMessage = objectMapper.readValue(in, DocumentRequest.class);
+        DocumentRequest ocrRequestMessage = objectMapper.readValue(in.getBody(), DocumentRequest.class);

-        long start = System.currentTimeMillis();
        log.info("Start ocr for file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());

-        setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+        try {
+            setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());

-        if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) {
-            byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
-            fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile);
+            if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) {
+                byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+                fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile);
+            }
+
+            try (var transferStream = new ByteArrayOutputStream()) {
+                ocrService.runOcrOnDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), transferStream);
+                try (var inputStream = new ByteArrayInputStream(transferStream.toByteArray())) {
+                    fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), inputStream);
+                }
+            } catch (IOException e) {
+                log.error("Failed to store file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+                throw new RuntimeException(e);
+            }
+
+            fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+        } catch (Exception e) {
+            log.warn("An exception occurred in ocr file stage: {}", e.getMessage());
+            in.getMessageProperties().getHeaders().put(X_ERROR_INFO_HEADER, e.getMessage());
+            in.getMessageProperties().getHeaders().put(X_ERROR_INFO_TIMESTAMP_HEADER, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+            throw e;
        }
-
-        var ocrResult = ocrService.ocrDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
-
-        fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), ocrResult);
-
-        long end = System.currentTimeMillis();
-        log.info("Successfully processed ocr for file with dossierId {} and fileId {}, took {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), end - start);
-
-        fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
    }


    @RabbitHandler
    @RabbitListener(queues = MessagingConfiguration.OCR_DLQ, concurrency = "1")
-    public void receiveOcrDQL(String in) throws JsonProcessingException {
+    public void receiveOcrDQL(Message failedMessage) throws IOException {

-        DocumentRequest ocrRequestMessage = objectMapper.readValue(in, DocumentRequest.class);
+        DocumentRequest ocrRequestMessage = objectMapper.readValue(failedMessage.getBody(), DocumentRequest.class);
        log.info("OCR DQL received: {}", ocrRequestMessage);
-        fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+        String errorMessage = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
+        OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
+        timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
+        fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(),
+                ocrRequestMessage.getFileId(),
+                new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp));
    }


@@ -1,10 +1,7 @@
 server:
-  port: 8086
+  port: 8097

-persistence-service.url: "http://persistence-service-v1:8080"
+persistence-service.url: "http://localhost:8085"
+tenant-user-management-service.url: "http://localhost:8091/internal"

-storage:
-  bucket-name: 'redaction'
-  endpoint: 'http://localhost:9000'
-  key: minioadmin
-  secret: minioadmin
+pdftron.license: demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a
@@ -2,6 +2,8 @@ info:
  description: OCR Service V1 Server

 persistence-service.url: "http://persistence-service-v1:8080"
+tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
+fforesight.tenants.remote: true

 server:
  port: 8080
@@ -26,24 +28,17 @@ spring:
          max-interval: 15000
        prefetch: 1

-platform.multi-tenancy:
-  enabled: false
-

 management:
  endpoint:
    metrics.enabled: ${monitoring.enabled:false}
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
-  endpoints.web.exposure.include: prometheus, health
+  endpoints.web.exposure.include: prometheus, health, metrics
  metrics.export.prometheus.enabled: ${monitoring.enabled:false}


 storage:
-  signer-type: 'AWSS3V4SignerType'
-  bucket-name: 'redaction'
-  region: 'us-east-1'
-  endpoint: 'https://s3.amazonaws.com'
  backend: 's3'

-pdftron.license: ${PDFTRON_LICENSE}
+pdftron.license: ${PDFTRON_LICENSE}
@@ -0,0 +1,95 @@
+package com.iqser.red.service.ocr.v1.server;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.service.ocr.v1.server.initializer.PDFNetInitializer;
+import com.iqser.red.service.ocr.v1.server.utils.FileSystemBackedStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.tenantcommons.TenantsClient;
+import com.pdftron.pdf.PDFNet;
+
+import jakarta.annotation.PostConstruct;
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import({AbstractTest.TestConfiguration.class, PDFNetInitializer.class})
+@AutoConfigureObservability
+public class AbstractTest {
+
+    @MockBean
+    private TenantsClient tenantsClient;
+
+    @Autowired
+    protected StorageService storageService;
+
+    @MockBean
+    protected RabbitTemplate rabbitTemplate;
+
+    private static String pdftronLicense;
+
+    @PostConstruct
+    @SneakyThrows
+    public void init() {
+        PDFNet.initialize(pdftronLicense);
+    }
+
+
+    @AfterAll
+    public static void terminatePDFNet() {
+        PDFNet.terminate();
+        System.out.println("PDFNet Terminated");
+    }
+
+
+    @SneakyThrows
+    public void dummyTest() {
+
+        // Build needs one test to not fail.
+        assertThat(1).isEqualTo(1);
+    }
+
+
+    @AfterEach
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
+    public static class TestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inMemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+}
@@ -1,112 +1,133 @@
 package com.iqser.red.service.ocr.v1.server;

+import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
 import static org.assertj.core.api.Assertions.assertThat;

-import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.concurrent.TimeUnit;

-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;

 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.ocr.v1.server.utils.FileSystemBackedStorageService;
-import com.iqser.red.service.ocr.v1.server.service.OCRService;
 import com.iqser.red.service.ocr.v1.server.service.FileStorageService;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.service.ocr.v1.server.service.OCRService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+import com.pdftron.pdf.OCRModule;

+import io.micrometer.prometheus.PrometheusMeterRegistry;
+import io.micrometer.prometheus.PrometheusTimer;
 import lombok.SneakyThrows;

-@ExtendWith(SpringExtension.class)
-@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT //
-        , properties = {"pdftron.ocrmodule.path=/YourOCRModulePath"})
-@Import(OcrServiceIntegrationTest.TestConfiguration.class)
-public class OcrServiceIntegrationTest {

-    @Autowired
-    protected StorageService storageService;
-
-    @Autowired
-    protected FileStorageService fileStorageService;
+@Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
+@SpringBootTest(properties = {"pdftron.ocrmodule.path=/YourOCRModulePath"})
+public class OcrServiceIntegrationTest extends AbstractTest {

    @Autowired
    protected ObjectMapper objectMapper;

-    @MockBean
-    protected RabbitTemplate rabbitTemplate;
-
    @Autowired
    private OCRService ocrService;

+    @Autowired
+    private PrometheusMeterRegistry registry;
+
+    @BeforeEach
+    @SneakyThrows
+    public void assertOCRModuleIsLoaded() {
+        assert OCRModule.isModuleAvailable();
+    }
+

    @Test
-    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
+    public void testOCRMetrics() {
+
+        testOCR("Watermark");
+        testOCR("Watermark");
+        testOCR("Watermark");
+
+        var ocrOnDocumentMeter = registry.getMeters().stream().filter(m -> m.getId().getName().equalsIgnoreCase("redactmanager_runOcrOnDocument")).findAny();
+        assertThat(ocrOnDocumentMeter.isPresent()).isTrue();
+        PrometheusTimer timer = (PrometheusTimer) ocrOnDocumentMeter.get();
+        assertThat(timer.count()).isEqualTo(3);
+        assertThat(timer.mean(TimeUnit.SECONDS)).isGreaterThan(0.1);
+    }
+
+
+    @Test
+    public void testOcr() {
+        // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
+        String text = testOCR("StitchedImagesMultiPage");
+    }
+
+
+    @Test
+    public void testManyRotatedImages() {
+        // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
+        String text = testOCR("manyRotatedImages");
+        assertThat(text).contains("Michela", "Gregori", "DVM", "PhD", "Pathologist");
+    }
+
+
+    @Test
+    public void testMergeImages() {
+        // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
+        String text = testOCR("merge_images");
+        assertThat(text).contains("Bodyweight change of dams with live young - group mean values",
+                "Control",
+                "mg/g day",
+                "10 mg/kg/day",
+                "20 mg/kg/",
+                "Days",
+                "50",
+                "-200",
+                "—250",
+                "150",
+                "200",
+                "250",
+                "—150");
+    }
+
+
+    @Test
+    public void testOCRWatermark() {
+
+        assertThat(testOCR("Watermark")).contains("syngenta");
+    }
+
+
+    @Test
+    public void testOCRInvisibleText() {
+
+        String text = testOCR("InvisibleText");
+        assertThat(text).contains("Michela", "Gregori", "DVM", "PhD", "Pathologist", "SIGNATURE PAGE");
+        assertThat(text).doesNotContain("COMPLETION DATE:", "LABORATORY PROJECT ID:", "AUTHOR(S):", "Substance");
+    }
+
+
    @SneakyThrows
-    public void testOCR() {
+    private String testOCR(String fileName) {

-        String fileName = "Watermark";
-
-        ClassPathResource imageInfoResource = new ClassPathResource("files/" + fileName + ".IMAGE_INFO.json");
        ClassPathResource pdfFileResource = new ClassPathResource("files/" + fileName + ".pdf");
-
        var originId = FileStorageService.getStorageId("dossier", "file", FileType.ORIGIN);
-        storageService.storeObject(originId, pdfFileResource.getInputStream());
-
-        var imageId = FileStorageService.getStorageId("dossier", "file", FileType.IMAGE_INFO);
-        storageService.storeObject(imageId, imageInfoResource.getInputStream());
-
-        var response = ocrService.ocrDocument("dossier", "file");
-
-        var out = FileUtils.openOutputStream(new File(getTemporaryDirectory() + "/" + fileName + ".pdf"));
-        IOUtils.copy(response, out);
-    }
-
-
-    @SneakyThrows
-    public void dummyTest() {
-
-        // Build needs one text to not fail.
-        assertThat(1).isEqualTo(1);
-    }
-
-
-    @AfterEach
-    public void cleanupStorage() {
-
-        if (this.storageService instanceof FileSystemBackedStorageService) {
-            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        try (var fileStream = pdfFileResource.getInputStream()) {
+            storageService.storeObject(TenantContext.getTenantId(), originId, fileStream);
        }
-    }
-
-
-    @Configuration
-    @EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
-    public static class TestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService();
+        try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
+            ocrService.runOcrOnDocument("dossier", "file", out);
+            System.out.println("File:" + getTemporaryDirectory() + "/" + fileName + ".pdf");
        }

+        try (var fileStream = new FileInputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
+            return extractAllTextFromDocument(fileStream);
+        }
    }
-
 }
@@ -0,0 +1,165 @@
+package com.iqser.red.service.ocr.v1.server.service;
+
+import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
+import static com.iqser.red.service.ocr.v1.server.utils.PdfDraw.drawGrid;
+import static com.iqser.red.service.ocr.v1.server.utils.PdfDraw.drawRectCollection;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import com.iqser.red.service.ocr.v1.server.AbstractTest;
+import com.pdftron.common.PDFNetException;
+import com.pdftron.pdf.ElementWriter;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.RectCollection;
+import com.pdftron.sdf.SDFDoc;
+
+import lombok.SneakyThrows;
+
+
+class ImagePositionRetrievalServiceTest extends AbstractTest {
+
+    @Autowired
+    private ImagePositionRetrievalService imagePositionRetrievalService;
+
+    @Test
+    @SneakyThrows
+    public void testImagePositionRetrievalForRotateTestFileWithImages() {
+
+        String fileName = "RotateTestFileWithImages";
+
+        List<int[]> allRectCoords = testImagePositionDetection(fileName);
+
+        assertThat(allRectCoords).contains(new int[]{48, 572, 295, 721},
+                new int[]{54, 279, 301, 428},
+                new int[]{360, 173, 509, 419},
+                new int[]{362, 522, 511, 768},
+                new int[]{459, 354, 608, 600},
+                new int[]{145, 404, 392, 553},
+                new int[]{151, 111, 398, 260},
+                new int[]{457, 5, 606, 251},
+                new int[]{395, 480, 545, 726},
+                new int[]{393, 130, 542, 377},
+                new int[]{88, 236, 334, 386},
+                new int[]{82, 530, 328, 679},
+                new int[]{465, 11, 614, 257},
+                new int[]{159, 117, 406, 266},
+                new int[]{467, 360, 617, 607},
+                new int[]{153, 410, 400, 559});
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testImagePositionRetrievalForRotateTestFileWithImagesExtremeCropbox() {
+
+        String fileName = "RotateTestFileWithImagesExtremeCropbox";
+
+        List<int[]> allRectCoords = testImagePositionDetection(fileName);
+        assertThat(allRectCoords).contains(new int[]{48, 572, 295, 721},
+                new int[]{362, 522, 511, 768},
+                new int[]{360, 173, 509, 419},
+                new int[]{54, 279, 301, 428},
+                new int[]{145, 192, 392, 341},
+                new int[]{459, 142, 608, 388},
+                new int[]{457, -207, 606, 39},
+                new int[]{151, -101, 398, 48},
+                new int[]{-30, 238, 216, 387},
+                new int[]{283, 188, 433, 434},
+                new int[]{281, -162, 430, 85},
+                new int[]{-24, -56, 222, 94},
+                new int[]{-39, 410, 208, 559},
+                new int[]{275, 360, 425, 607},
+                new int[]{273, 11, 422, 257},
+                new int[]{-33, 117, 214, 266});
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testMergeImages() {
+
+        String fileName = "merge_images";
+        List<int[]> allRectCoords = testImagePositionDetection(fileName);
+        assertThat(allRectCoords).contains(new int[]{90, 284, 398, 770});
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testStitchedImagesMultiPage() {
+
+        String fileName = "StitchedImagesMultiPage";
+        List<int[]> allRectCoords = testImagePositionDetection(fileName);
+        assertThat(allRectCoords.size()).isEqualTo(48);
+    }
+
+    @Test
+    @SneakyThrows
+    public void testEveryPointInDashedLineIsImage() {
+        String fileName = "everyPointInDashedLineIsImage";
+        List<int[]> allRectCoords = testImagePositionDetection(fileName);
+        assertThat(allRectCoords.size()).isEqualTo(0);
+    }
+
+
+    private List<int[]> testImagePositionDetection(String fileName) throws IOException, PDFNetException {
+
+        try (InputStream fileStream = new FileInputStream(new ClassPathResource("files/" + fileName + ".pdf").getFile().getAbsolutePath())) {
+            PDFDoc pdfDoc = new PDFDoc(fileStream);
+
+            Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, false);
+
+            ElementWriter writer = new ElementWriter();
+            pageIdToRectCollection.forEach((pageId, rectCollection) -> {
+                try {
+                    writer.begin(pdfDoc.getPage(pageId));
+                    drawRectCollection(writer, rectCollection);
+                    drawGrid(writer, pdfDoc.getPage(pageId));
+                    writer.end();
+                    StringBuilder zonesString = new StringBuilder();
+                    for (int j = 0; j < rectCollection.getNumRects(); ++j) {
+                        var r = rectCollection.getRectAt(j);
+                        zonesString.append(String.format("%d:[lower left (%.1f|%.1f) upper right (%.1f|%.1f)]; ", j, r.getX1(), r.getY1(), r.getX2(), r.getY2()));
+                    }
+                    System.out.printf("Page %d: rotation: %d OCR regions %s \n", pageId, pdfDoc.getPage(pageId).getRotation(), zonesString);
+                } catch (PDFNetException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+
+            // Check visually for red Rectangles to match images in the saved pdf file
+            try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + "_IMAGE_BBOX.pdf")) {
+                out.write(pdfDoc.save(SDFDoc.SaveMode.LINEARIZED, null));
+            }
+            pdfDoc.close();
+            System.out.println("result file: " + getTemporaryDirectory() + "/" + fileName + "_IMAGE_BBOX.pdf");
+            // round all coords to nearest int to account for inconsistencies with the calculation of the bounding box
+            return pageIdToRectCollection.values().stream().map(this::toRoundedCoordinateArrayList).flatMap(List::stream).collect(Collectors.toList());
+        }
+    }
+
+
+    @SneakyThrows
+    private List<int[]> toRoundedCoordinateArrayList(RectCollection rectCollection) {
+
+        List<int[]> coords = new ArrayList<>(rectCollection.getNumRects());
+        for (int i = 0; i < rectCollection.getNumRects(); ++i) {
+            var r = rectCollection.getRectAt(i);
+            coords.add(new int[]{(int) Math.round(r.getX1()), (int) Math.round(r.getY1()), (int) Math.round(r.getX2()), (int) Math.round(r.getY2())});
+        }
+        return coords;
+    }
+
+}
@@ -0,0 +1,51 @@
+package com.iqser.red.service.ocr.v1.server.service;
+
+import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
+import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.ClassPathResource;
+
+import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
+import com.iqser.red.service.ocr.v1.server.AbstractTest;
+
+import lombok.SneakyThrows;
+
+public class InvisibleElementRemovalServiceTest extends AbstractTest {
+
+   @Autowired
+   private InvisibleElementRemovalService invisibleElementRemovalService;
+
+
+    @Test
+    @SneakyThrows
+    public void testRemoveInvisibleText() {
+
+        String fileName = "InvisibleText";
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/" + fileName + ".pdf");
+
+        try (var initialFileStream = pdfFileResource.getInputStream(); var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
+            invisibleElementRemovalService.removeInvisibleElements(initialFileStream, out, false);
+        }
+
+        try (var initialFileStream = pdfFileResource.getInputStream(); var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + "_delta.pdf")) {
+            invisibleElementRemovalService.removeInvisibleElements(initialFileStream, out, true);
+        }
+
+        System.out.println("Output File without invisible elements: files/" + fileName + ".pdf");
+        System.out.println("Output Delta File without invisible elements: files/" + fileName + "_delta.pdf");
+
+        try (var fileStream = new FileInputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
+            String[] text = extractAllTextFromDocument(fileStream).split("\n");
+            assertThat(text).contains("APPENDIX 16 Pathology Report", "Amendment 1", "Page 255 of 260");
+        }
+
+    }
+}
@@ -32,7 +32,7 @@ public class FileSystemBackedStorageService implements StorageService {

    @SneakyThrows
    @Override
-    public InputStreamResource getObject(String objectId) {
+    public InputStreamResource getObject(String tenantId, String objectId) {

        var res = dataMap.get(objectId);
        if (res == null) {
@@ -44,28 +44,22 @@ public class FileSystemBackedStorageService implements StorageService {


    @Override
-    public void deleteObject(String objectId) {
+    public void deleteObject(String tenantId, String objectId) {

        dataMap.remove(objectId);
    }


    @Override
-    public boolean objectExists(String objectId) {
+    public boolean objectExists(String tenantId, String objectId) {

        return dataMap.containsKey(objectId);
    }


-    @Override
-    public void init() {
-
-    }
-
-
    @Override
    @SneakyThrows
-    public <T> void storeJSONObject(String objectId, T any) {
+    public <T> void storeJSONObject(String tenantId, String objectId, T any) {

        File tempFile = File.createTempFile("test", ".tmp");
        getMapper().writeValue(new FileOutputStream(tempFile), any);
@@ -81,7 +75,7 @@ public class FileSystemBackedStorageService implements StorageService {

    @Override
    @SneakyThrows
-    public <T> T readJSONObject(String objectId, Class<T> clazz) {
+    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {

        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
            throw new StorageObjectDoesNotExist("Stored object not found");
@@ -104,7 +98,7 @@ public class FileSystemBackedStorageService implements StorageService {

    @Override
    @SneakyThrows
-    public void storeObject(String objectId, InputStream stream) {
+    public void storeObject(String tenantId, String objectId, InputStream stream) {

        File tempFile = File.createTempFile("test", ".tmp");

@@ -0,0 +1,74 @@
+package com.iqser.red.service.ocr.v1.server.utils;
+
+import com.pdftron.pdf.ColorPt;
+import com.pdftron.pdf.ColorSpace;
+import com.pdftron.pdf.Element;
+import com.pdftron.pdf.ElementBuilder;
+import com.pdftron.pdf.ElementWriter;
+import com.pdftron.pdf.Page;
+import com.pdftron.pdf.Rect;
+import com.pdftron.pdf.RectCollection;
+
+import lombok.SneakyThrows;
+
+
+public class PdfDraw {
+
+    @SneakyThrows
+    public static void drawGrid(ElementWriter writer, Page page) {
+
+        ElementBuilder eb = new ElementBuilder();
+        double dX = 15;
+        double dY = 15;
+        int nRows = (int) (page.getPageHeight() / dY) + 1;
+        int nCols = (int) (page.getPageWidth() / dX) + 1;
+        for (int row = 0; row < nRows; ++row) {
+            for (int col = 0; col < nCols; ++col) {
+                Element cell = eb.createRect(col * dX, row * dY, dX, dY);
+                cell.setPathStroke(true);
+                cell.getGState().setLineWidth(1);
+                cell.getGState().setStrokeOpacity(0.1);
+                cell.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+                if (row == 0 && col == 0) {
+                    cell.getGState().setStrokeColor(new ColorPt(0, 0, 1));
+                    cell.setPathFill(true);
+                    cell.getGState().setFillOpacity(0.8);
+                    cell.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
+                    cell.getGState().setFillColor(new ColorPt(0, 0, 1));
+                } else {
+                    cell.setPathFill(false);
+                    cell.getGState().setStrokeColor(new ColorPt(0.1, 0.1, 0.1));
+                }
+                writer.writePlacedElement(cell);
+            }
+        }
+        eb.destroy();
+    }
+
+
+    @SneakyThrows
+    public static void drawRectCollection(ElementWriter writer, RectCollection rectCollection) {
+
+        ColorPt colorPt = new ColorPt(1, 0, 0);
+        ElementBuilder eb = new ElementBuilder();
+        for (int i = 0; i < rectCollection.getNumRects(); ++i) {
+            Rect r = rectCollection.getRectAt(i);
+            Element rect = eb.createRect(r.getX1(), r.getY1(), r.getWidth(), r.getHeight());
+
+            rect.setPathStroke(true);
+            rect.getGState().setLineWidth(5);
+            rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+            rect.getGState().setStrokeColor(colorPt);
+
+            rect.setPathFill(true);
+            rect.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
+            rect.getGState().setFillColor(colorPt);
+            rect.getGState().setFillOpacity(0.5);
+
+            writer.writePlacedElement(rect);
+        }
+        colorPt.destroy();
+        eb.destroy();
+    }
+
+}
@@ -5,4 +5,13 @@ spring:
    allow-circular-references: true # FIXME


-pdftron.license: demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a
+pdftron.license: demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a
+
+
+management:
+  endpoint:
+    metrics.enabled: true
+    prometheus.enabled: true
+    health.enabled: true
+  endpoints.web.exposure.include: prometheus, health, metrics
+  metrics.export.prometheus.enabled: true
@@ -1 +0,0 @@
-{"dossierId": "55547c91-6b0e-4aa6-9009-2e7c4cd90f13", "fileId": "917b9d9c9f548f85fef3679db45ff46c", "targetFileExtension": "ORIGIN.pdf.gz", "responseFileExtension": "IMAGE_INFO.json.gz", "data": [{"classification": {"label": "signature", "probabilities": {"signature": 0.9927, "logo": 0.0038, "other": 0.0034, "formula": 0.0}}, "representation": "FFF2CF0F7C74FFC1070830FFF", "position": {"x1": -7, "x2": 603, "y1": 0, "y2": 852, "pageNumber": 1}, "geometry": {"width": 610, "height": 852}, "alpha": false, "filters": {"geometry": {"imageSize": {"quotient": 1.0096, "tooLarge": true, "tooSmall": false}, "imageFormat": {"quotient": 0.716, "tooTall": false, "tooWide": false}}, "probability": {"unconfident": false}, "allPassed": false}}], "dataCV": []}
@@ -1 +0,0 @@
-{"dossierId": "55547c91-6b0e-4aa6-9009-2e7c4cd90f13", "fileId": "d7f1e0e37cba4e28ebdf894a79d3bd67", "targetFileExtension": "ORIGIN.pdf.gz", "responseFileExtension": "IMAGE_INFO.json.gz", "data": [{"classification": {"label": "signature", "probabilities": {"signature": 0.9872, "logo": 0.0064, "other": 0.0063, "formula": 0.0001}}, "representation": "FFFCF10608F6F89747BFFC301", "position": {"x1": -9, "x2": 584, "y1": 9, "y2": 849, "pageNumber": 1}, "geometry": {"width": 593, "height": 840}, "alpha": false, "filters": {"geometry": {"imageSize": {"quotient": 0.9992, "tooLarge": true, "tooSmall": false}, "imageFormat": {"quotient": 0.706, "tooTall": false, "tooWide": false}}, "probability": {"unconfident": false}, "allPassed": false}}], "dataCV": []}
@@ -1 +0,0 @@
-{"dossierId": "55547c91-6b0e-4aa6-9009-2e7c4cd90f13", "fileId": "32b19ec38896f5105c09041def470c90", "targetFileExtension": "ORIGIN.pdf.gz", "responseFileExtension": "IMAGE_INFO.json.gz", "data": [{"classification": {"label": "logo", "probabilities": {"logo": 0.9999, "signature": 0.0001, "formula": 0.0, "other": 0.0}}, "representation": "307EF8F6E9833CE9D7AF9EFFF", "position": {"x1": 26, "x2": 586, "y1": -2, "y2": 794, "pageNumber": 1}, "geometry": {"width": 560, "height": 796}, "alpha": false, "filters": {"geometry": {"imageSize": {"quotient": 0.959, "tooLarge": true, "tooSmall": false}, "imageFormat": {"quotient": 0.7035, "tooTall": false, "tooWide": false}}, "probability": {"unconfident": false}, "allPassed": false}}, {"classification": {"label": "logo", "probabilities": {"logo": 1.0, "formula": 0.0, "other": 0.0, "signature": 0.0}}, "representation": "FFF7FFD2000000018F3FFEFFF", "position": {"x1": 90, "x2": 210, "y1": 676, "y2": 720, "pageNumber": 1}, "geometry": {"width": 120, "height": 44}, "alpha": false, "filters": {"geometry": {"imageSize": {"quotient": 0.1044, "tooLarge": false, "tooSmall": false}, "imageFormat": {"quotient": 2.7273, "tooTall": false, "tooWide": false}}, "probability": {"unconfident": false}, "allPassed": true}}], "dataCV": []}
@@ -0,0 +1,16 @@
+<Configuration>
+
+    <Appenders>
+        <Console name="CONSOLE" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="warn">
+            <AppenderRef ref="CONSOLE"/>
+        </Root>
+        <Logger name="com.iqser" level="info"/>
+    </Loggers>
+
+</Configuration>
@@ -7,13 +7,13 @@
    <parent>
        <groupId>com.iqser.red</groupId>
        <artifactId>platform-dependency</artifactId>
-        <version>1.14.0</version>
+        <version>2.2.0</version>
        <relativePath/>
    </parent>

    <groupId>com.iqser.red.service</groupId>
    <artifactId>ocr-service-v1</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>3.0-SNAPSHOT</version>

    <packaging>pom</packaging>

@@ -27,7 +27,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.20.0</version>
+                <version>2.1.0</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -39,12 +39,12 @@
                <plugin>
                    <groupId>org.sonarsource.scanner.maven</groupId>
                    <artifactId>sonar-maven-plugin</artifactId>
-                    <version>3.9.0.2155</version>
+                    <version>3.9.1.2184</version>
                </plugin>
                <plugin>
                    <groupId>org.owasp</groupId>
                    <artifactId>dependency-check-maven</artifactId>
-                    <version>6.3.1</version>
+                    <version>8.1.2</version>
                    <configuration>
                        <format>ALL</format>
                    </configuration>
@@ -6,12 +6,11 @@

    <groupId>com.iqser.red.service</groupId>
    <artifactId>ocr-service</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>3.0-SNAPSHOT</version>

    <packaging>pom</packaging>

    <modules>
-        <module>bamboo-specs</module>
        <module>ocr-service-v1</module>
        <module>ocr-service-image-v1</module>
    </modules>
Author	SHA1	Message	Date
Timo Bejan	038cc22f28	Merge branch 'RED-6686' into 'master' RED-6686 Extract Tenant and user-management code into a separate service. See merge request redactmanager/ocr-service!1	2023-06-26 21:58:50 +02:00
Timo Bejan	06a504cf2f	RED-6686 Extract Tenant and user-management code into a separate service.	2023-06-26 21:58:49 +02:00
Raphael Arnold	eb91574b0a	Merge branch 'RED-6965' into 'master' RED-6965: Null pointer removal Closes RED-6965 See merge request redactmanager/ocr-service!3	2023-06-21 16:15:17 +02:00
RaphaelArnold	b28cc721c2	RED-6965: Null pointer removal	2023-06-21 16:11:07 +02:00
Dominique Eifländer	7dcc59c750	Merge branch 'RED-6072-timestamp' into 'master' RED-6072 - As Operation I want to see why files are in an ERROR state Closes RED-6072 See merge request redactmanager/ocr-service!2	2023-06-15 14:08:39 +02:00
Corina Olariu	b15bcfa372	RED-6072 - As Operation I want to see why files are in an ERROR state - add timestamp to error info	2023-06-15 14:37:32 +03:00
Christoph Schabert	d2f38d7cf0	Update 2 files - /.gitlab-ci.yml - /pom.xml	2023-05-31 15:44:02 +02:00
Christoph Schabert	df59fd83ed	Update 6 files - /bamboo-specs/src/main/java/buildjob/PlanSpec.java - /bamboo-specs/src/main/resources/scripts/build-java.sh - /bamboo-specs/src/main/resources/scripts/sonar-java.sh - /bamboo-specs/src/test/java/buildjob/PlanSpecTest.java - /bamboo-specs/pom.xml - /.gitlab-ci.yml	2023-05-31 15:10:37 +02:00
Corina Olariu	e400a04644	Pull request #22 : RED-6072 Merge in RED/ocr-service from RED-6072 to master * commit '961129b917e92b4dda2085215d3f15e205ec35a3': RED-6072 - As Operation I want to see why files are in an ERROR state - update the ocr failed with error information - in case of exception the reason is caught and added tot the message received from the queue RED-6072 - As Operation I want to see why files are in an ERROR state update from iqser to knecon	2023-05-25 14:59:04 +02:00
Dominique Eiflaender	933b4bd3de	Pull request #23 : RED-5567: Upgraded to lastest pdftron version Merge in RED/ocr-service from RED-5567-exptest to master * commit 'ee94f1c711c923e865e559499674c1658c87fbb4': RED-5567: Upgraded to lastest pdftron version	2023-05-25 14:19:18 +02:00
deiflaender	ee94f1c711	RED-5567: Upgraded to lastest pdftron version	2023-05-25 14:14:59 +02:00
devplant	961129b917	RED-6072 - As Operation I want to see why files are in an ERROR state - update the ocr failed with error information - in case of exception the reason is caught and added tot the message received from the queue	2023-05-25 14:29:40 +03:00
devplant	0307da9093	RED-6072 - As Operation I want to see why files are in an ERROR state update from iqser to knecon	2023-05-25 14:20:14 +03:00
Corina Olariu	aac56cc2ed	Pull request #21 : RED-5694 - Upgrade spring-boot to 3.0 Merge in RED/ocr-service from RED-5694-storage to master * commit '828b4f53c837a77b860dc3df20a8e65b1330eaa8': RED-5694 - Upgrade spring-boot to 3.0 - add -DknownExploitedEnabled=false - import StorageAutoConfiguration to Application	2023-04-21 14:08:04 +02:00
devplant	828b4f53c8	RED-5694 - Upgrade spring-boot to 3.0 - add -DknownExploitedEnabled=false - import StorageAutoConfiguration to Application	2023-04-20 16:14:28 +03:00
Corina Olariu	062cf46e30	Pull request #20 : RED-5694 Merge in RED/ocr-service from RED-5694 to master * commit 'f72001c4ac0431c482cc467e948bdc972643d785': RED-5694 - Upgrade spring-boot to 3.0 - remove commented code RED-5694 - Upgrade spring-boot to 3.0 - remove versions already defined in spring boot RED-5694 - Upgrade spring-boot to 3.0 - remove unused import RED-5694 - Upgrade spring-boot to 3.0 - update platform-dependency - update other dependencies versions to the latest	2023-04-06 12:42:23 +02:00
devplant	f72001c4ac	RED-5694 - Upgrade spring-boot to 3.0 - remove commented code	2023-04-06 13:36:43 +03:00
devplant	81fd35f9c0	RED-5694 - Upgrade spring-boot to 3.0 - remove versions already defined in spring boot	2023-04-06 13:35:42 +03:00
devplant	17b1932926	RED-5694 - Upgrade spring-boot to 3.0 - remove unused import	2023-04-06 11:57:36 +03:00
devplant	c00e9290bc	RED-5694 - Upgrade spring-boot to 3.0 - update platform-dependency - update other dependencies versions to the latest	2023-04-06 11:52:44 +03:00
Thomas Beyer	d9e4f79099	Pull request #19 : RED-4875 1 Merge in RED/ocr-service from RED-4875_1 to master * commit '036203c24a1f5eb1588e945eb020a666cac5dba2': RED-4875 - delete commented out classes RED-4875 - set version of pdftron-common-logics to newest (release) RED-4875 - removed duration log RED-4875 - update version of pdftron-commons to newest RED-4875 - update version of pdftron logic commons to newest RED-4875 - set version of common pdftron logics to newest and move PdfTextExtraction to this new repo RED-4875 - update version of pdftron-logic-commons to newest RED-4875 - call logic of new repo pdftron-logic-commons instead of local one	2023-03-22 12:07:24 +01:00
Thomas Beyer	036203c24a	RED-4875 - delete commented out classes	2023-03-21 18:20:26 +01:00
Thomas Beyer	6ac71f6d94	RED-4875 - set version of pdftron-common-logics to newest (release)	2023-03-21 17:00:41 +01:00
Thomas Beyer	dd12611fdc	RED-4875 - removed duration log	2023-03-21 13:18:41 +01:00
Thomas Beyer	b0c4c25bec	RED-4875 - update version of pdftron-commons to newest	2023-03-21 12:28:57 +01:00
Thomas Beyer	5efa0e96a8	RED-4875 - update version of pdftron logic commons to newest	2023-03-20 11:25:38 +01:00
Thomas Beyer	fd92419895	RED-4875 - set version of common pdftron logics to newest and move PdfTextExtraction to this new repo	2023-03-20 10:01:33 +01:00
Thomas Beyer	142e8cf957	RED-4875 - update version of pdftron-logic-commons to newest	2023-03-17 17:25:52 +01:00
Thomas Beyer	143538fa40	RED-4875 - call logic of new repo pdftron-logic-commons instead of local one	2023-03-17 10:33:48 +01:00
Christoph Schabert	74a094b42d	Change Tagging	2023-03-13 09:41:59 +01:00
Timo Bejan	8ab4092046	Pull request #18 : RED-6162 - bumped version Merge in RED/ocr-service from RED-6162 to master * commit 'c86cabf70a9ec90687c3f123768f2293da49c4ff': RED-6162 - bumped version RED-6162 - bumped version	2023-03-10 21:45:38 +01:00
Timo Bejan	c86cabf70a	RED-6162 - bumped version	2023-03-10 22:43:35 +02:00
Dominique Eiflaender	936a795797	Pull request #17 : RED-4645: Multitenancy for storage Merge in RED/ocr-service from RED-4645 to master * commit 'e5df0ec65864667134affad2af2a5118126355ce': RED-4645: Multitenancy for storage	2023-03-10 16:42:43 +01:00
deiflaender	e5df0ec658	RED-4645: Multitenancy for storage	2023-03-10 16:31:34 +01:00
Timo Bejan	0d9365d020	RED-6162 - bumped version	2023-03-10 15:59:25 +02:00
Timo Bejan	5003970fe1	Pull request #16 : RED-6162 - bumped version Merge in RED/ocr-service from RED-6162 to master * commit '9c6fa4384a7ea43dfd36c6fcb6639e5127ab92b9': RED-6162 - bumped version	2023-03-10 14:56:05 +01:00
Timo Bejan	9c6fa4384a	RED-6162 - bumped version	2023-03-10 15:52:37 +02:00
Timo Bejan	444b823cce	Pull request #15 : RED-6162 Redaction Gateway - Persistence Service Merge Updates Merge in RED/ocr-service from RED-6162 to master * commit '6208ff029252884ced189806acd4118cd21530e1': RED-6182 - version bump RED-6162 - test issue on bamboo RED-6162 - test issue on bamboo RED-6162 - persistence update - identity test RED-6162 - persistence update - reverse dependency cleanup	2023-03-10 11:27:13 +01:00
Timo Bejan	6208ff0292	RED-6182 - version bump	2023-03-10 08:45:48 +02:00
Timo Bejan	5156c560f2	RED-6162 - test issue on bamboo	2023-03-10 08:45:48 +02:00
Timo Bejan	c2a7e32789	RED-6162 - test issue on bamboo	2023-03-10 08:45:48 +02:00
Timo Bejan	c1cd4d1cab	RED-6162 - persistence update - identity test	2023-03-10 08:45:48 +02:00
Timo Bejan	da5a1cc042	RED-6162 - persistence update - reverse dependency cleanup	2023-03-10 08:45:48 +02:00
Kilian Schuettler	d91fb737cb	Pull request #14 : RED-6321: OCR not working correctly with 3.6.0 Merge in RED/ocr-service from RED-6321 to master * commit '2d503c74a6fba0b02ac4c92af82a493165e45761': RED-6321: OCR not working correctly with 3.6.0 added end() statements to formWriters to write their changes to the PDF contentStream moved replaceOriginalPageWithOcrPage outside the main OCR Loop since it caused exponential RAM Usage in some cases	2023-03-06 10:16:02 +01:00
Kilian Schuettler	2d503c74a6	RED-6321: OCR not working correctly with 3.6.0 added end() statements to formWriters to write their changes to the PDF contentStream moved replaceOriginalPageWithOcrPage outside the main OCR Loop since it caused exponential RAM Usage in some cases	2023-03-06 10:08:32 +01:00
Corina Olariu	bdcba7cb6d	Pull request #11 : RED-4988 Check jacoco version in poms and update to a current compatible version Merge in RED/ocr-service from RED-4988 to master * commit '334409e4db326945a1eeb86004d79e0b0faeb99c': RED-4988 Check jacoco version in poms and update to a current compatible version - update platform-dependency, platform-commons-dependency	2023-03-02 09:28:11 +01:00
Shamel Hussain	23622d4a85	Pull request #13 : RED-5718: Update base image version to allow using random user ids Merge in RED/ocr-service from shussain/Dockerfile-1677589647499 to master * commit '4266184456ea4d97fa19b52ff37d5d9fc09dd170': RED-5718: Update base image version to allow using random user ids	2023-02-28 14:56:37 +01:00
Shamel Hussain	4266184456	RED-5718: Update base image version to allow using random user ids	2023-02-28 14:07:44 +01:00
Kilian Schuettler	a4ca2db37d	Pull request #12 : RED-6280: Performance Test Issue with OCR-Service Merge in RED/ocr-service from RED-6280 to master * commit '742725834933ad74ad582366b2b62015524bedb3': RED-6280: Performance Test Issue with OCR-Service removed init/terminate calls again manual memory cleanup at every opportunity	2023-02-28 10:30:58 +01:00
Kilian Schuettler	7427258349	RED-6280: Performance Test Issue with OCR-Service removed init/terminate calls again manual memory cleanup at every opportunity	2023-02-28 10:21:21 +01:00
devplant	334409e4db	RED-4988 Check jacoco version in poms and update to a current compatible version - update platform-dependency, platform-commons-dependency	2023-02-27 17:41:15 +02:00
Kilian Schuettler	bc661b7ea4	Pull request #9 : RED-6126: In the OCRService, OCR Text is not applied to Document Merge in RED/ocr-service from RED-6126 to master * commit 'caff5580dda644451433921a6a66dafe1cfa5dca': RED-6126: In the OCRService, OCR Text is not applied to Document refactored Tests with inheritance called PDFNet init/terminate in tests don't call init on startup RED-6126: In the OCRService, OCR Text is not applied to Document called PDFNet.initialize and terminate before and after message receive updated comments renamed some variables	2023-02-22 13:33:49 +01:00
Kilian Schuettler	caff5580dd	RED-6126: In the OCRService, OCR Text is not applied to Document refactored Tests with inheritance called PDFNet init/terminate in tests *don't call init on startup	2023-02-22 11:58:07 +01:00
Kilian Schuettler	430ad45a67	RED-6126: In the OCRService, OCR Text is not applied to Document called PDFNet.initialize and terminate before and after message receive updated comments *renamed some variables	2023-02-22 10:32:55 +01:00
Kilian Schuettler	a6d99f5916	Pull request #8 : RED-6126: In the OCRService, OCR Text is not applied to Document Merge in RED/ocr-service from RED-6126 to master * commit '0bc4fea2a52c92efaaaf8cf93c2ae02766168a80': RED-6126: In the OCRService, OCR Text is not applied to Document *removed unnecessary getXObject() call, since it fails for inline_images	2023-02-14 09:57:28 +01:00
Kilian Schuettler	0bc4fea2a5	RED-6126: In the OCRService, OCR Text is not applied to Document *removed unnecessary getXObject() call, since it fails for inline_images	2023-02-13 17:55:02 +01:00
Kilian Schuettler	001719a34c	Pull request #7 : RED-6126 performance test Merge in RED/ocr-service from RED-6126-performance-test to master * commit '37f1e03ebcd5356e0f0b403a5c0cdd20fc133997': RED-6126: performance-test refactor to improve cleanness closed inputStream RED-6126: performance-test fixed NullPointerException fixed StackOverFlowError by ignoring very small images and moving to while loop instead of recursion RED-6126: performance-test fixed time calculation RED-6126: performance-test improved error logging RED-6126: performance-test re-enabled overlap detection re-creating helper document for every page instead of reusing and adding/removing pages RED-6126: Performance Tests moved to streams for pdf file transfer disabled overlap detection	2023-02-10 15:00:55 +01:00
Kilian Schuettler	37f1e03ebc	RED-6126: performance-test refactor to improve cleanness closed inputStream	2023-02-10 14:49:10 +01:00
Kilian Schuettler	b3fa14b342	RED-6126: performance-test fixed NullPointerException fixed StackOverFlowError by ignoring very small images and moving to while loop instead of recursion	2023-02-10 12:27:16 +01:00
Kilian Schuettler	7065d098f3	RED-6126: performance-test *fixed time calculation	2023-02-09 16:31:42 +01:00
Kilian Schuettler	8db0b712f7	RED-6126: performance-test *improved error logging	2023-02-09 13:57:21 +01:00
Kilian Schuettler	6ccf3f80fc	RED-6126: performance-test re-enabled overlap detection re-creating helper document for every page instead of reusing and adding/removing pages	2023-02-09 11:22:39 +01:00
Kilian Schuettler	e705f869fd	RED-6126: Performance Tests moved to streams for pdf file transfer disabled overlap detection	2023-02-09 11:09:52 +01:00
Timo Bejan	efaa291e43	Pull request #6 : RED-4609 - added ocr metric, enabled prometheus, added test for metric Merge in RED/ocr-service from RED-4609 to master * commit '7c71d8ad041f839c21ec26023ee8eaef670a4924': RED-4609 - added ocr metric, enabled prometheus, added test for metric	2023-02-09 10:57:37 +01:00
Timo Bejan	7c71d8ad04	RED-4609 - added ocr metric, enabled prometheus, added test for metric	2023-02-08 16:46:51 +02:00
Kilian Schuettler	b0a658213d	Pull request #5 : RED-6126 Merge in RED/ocr-service from RED-6126 to master * commit '00cfe9e44948c153857ad59442dbc9349e1d4555': RED-6126: In the OCRService, OCR Text is not applied to Document reformatted InvisibleElementRemovalService with new Code Style RED-6126: In the OCRService, OCR Text is not applied to Document updated some comments very slight refactor RED-6126: In the OCRService, OCR Text is not applied to Document complete refactor of the OCRService moved image position retrieval to new class instead of image service added new tests for image rotation RED-6126: In the OCRService, OCR Text is not applied to Document removed private configuration RED-6126: In the OCRService, OCR Text is not applied to Document formatted one line RED-6126: In the OCRService, OCR Text is not applied to Document reverted application of OCR Text to Document to old state refactored OCR Service slightly *added meaningful test cases	2023-02-07 13:35:32 +01:00
Kilian Schuettler	00cfe9e449	RED-6126: In the OCRService, OCR Text is not applied to Document *reformatted InvisibleElementRemovalService with new Code Style	2023-02-07 12:52:09 +01:00
Kilian Schuettler	d0d6bf70a4	RED-6126: In the OCRService, OCR Text is not applied to Document updated some comments very slight refactor	2023-02-07 12:09:04 +01:00
Kilian Schuettler	a415224db5	RED-6126: In the OCRService, OCR Text is not applied to Document complete refactor of the OCRService moved image position retrieval to new class instead of image service *added new tests for image rotation	2023-02-07 12:05:24 +01:00
Kilian Schuettler	355887c865	RED-6126: In the OCRService, OCR Text is not applied to Document *removed private configuration	2023-02-03 13:16:56 +01:00
Kilian Schuettler	ab566a11a9	RED-6126: In the OCRService, OCR Text is not applied to Document *formatted one line	2023-02-03 13:03:47 +01:00
Kilian Schuettler	edd044395e	RED-6126: In the OCRService, OCR Text is not applied to Document reverted application of OCR Text to Document to old state refactored OCR Service slightly *added meaningful test cases	2023-02-03 13:01:01 +01:00
Kilian Schuettler	b37ec5afc9	Pull request #4 : RED-6019: Remove hidden text when processing OCR Merge in RED/ocr-service from RED-6019 to master * commit 'a96260f77fd5b546a5d27d84f34861742f13ddff': RED-6019: Remove hidden text when processing OCR moved InvisibleElementRemovalDto to private inner record of InvisibleElementRemovalService added comments for color choices RED-6019: Remove hidden text when processing OCR moved to release version of platform-dependencies restored annotationProcessors RED-6019: Remove hidden text when processing OCR code refactor upgrade to java 17 RED-6019: Remove hidden text when processing OCR handled cases: Text which is transparent or is set to not render Elements outside of clipping path Elements that have been painted over by visible and filled Paths unhandled cases: Elements covered by widely stroked path Elements same color as background Any Text set to clipping with its many interactions with other elements	2023-02-02 13:05:03 +01:00
Kilian Schuettler	a96260f77f	RED-6019: Remove hidden text when processing OCR moved InvisibleElementRemovalDto to private inner record of InvisibleElementRemovalService added comments for color choices	2023-02-02 13:01:58 +01:00
Kilian Schuettler	12fbdbee50	RED-6019: Remove hidden text when processing OCR moved to release version of platform-dependencies restored annotationProcessors	2023-02-02 10:53:19 +01:00
Kilian Schuettler	99a0cb51d0	RED-6019: Remove hidden text when processing OCR code refactor upgrade to java 17	2023-02-02 10:27:01 +01:00
Kilian Schuettler	fd7ec6e7aa	RED-6019: Remove hidden text when processing OCR handled cases: Text which is transparent or is set to not render Elements outside of clipping path Elements that have been painted over by visible and filled Paths unhandled cases: Elements covered by widely stroked path Elements same color as background Any Text set to clipping with its many interactions with other elements	2023-01-30 16:13:51 +01:00
Dominique Eiflaender	265fac8099	Pull request #3 : RED-5911: Reverted to old ocr logic that uses ContentReplacer/TextExtractor to remove text behind images Merge in RED/ocr-service from RED-5911 to master * commit '7a4c5c2f898e83623a66ef29ab9ed696e2057e24': RED-5911: Reverted to old ocr logic that uses ContentReplacer/TextExtractor to remove text behind images	2023-01-17 12:42:10 +01:00
deiflaender	7a4c5c2f89	RED-5911: Reverted to old ocr logic that uses ContentReplacer/TextExtractor to remove text behind images	2023-01-17 12:15:34 +01:00
Philipp Schramm	e535861da8	Pull request #2 : RED-5911 Bugfix for removed texts within tables Merge in RED/ocr-service from RED-5911 to master * commit '0e8dfed4410b28a6316f8e328fc166339852565f': RED-5911 Bugfix for removed texts within tables	2023-01-04 15:02:15 +01:00
Philipp Schramm	0e8dfed441	RED-5911 Bugfix for removed texts within tables	2023-01-04 12:31:51 +01:00
Dominique Eiflaender	2fe0eb024a	Pull request #1 : RSS-146: Install Ghostscript to imporve ocr quality Merge in RED/ocr-service from RSS-146 to master * commit '37548ddbf89c0ca5e25d18b19af9400ef5e608c0': RSS-146: Install Ghostscript to imporve ocr quality	2022-12-14 17:41:01 +01:00
deiflaender	37548ddbf8	RSS-146: Install Ghostscript to imporve ocr quality	2022-12-14 16:11:00 +01:00
				`@@ -1 +0,0 @@`
				{"dossierId": "55547c91-6b0e-4aa6-9009-2e7c4cd90f13", "fileId": "917b9d9c9f548f85fef3679db45ff46c", "targetFileExtension": "ORIGIN.pdf.gz", "responseFileExtension": "IMAGE_INFO.json.gz", "data": [{"classification": {"label": "signature", "probabilities": {"signature": 0.9927, "logo": 0.0038, "other": 0.0034, "formula": 0.0}}, "representation": "FFF2CF0F7C74FFC1070830FFF", "position": {"x1": -7, "x2": 603, "y1": 0, "y2": 852, "pageNumber": 1}, "geometry": {"width": 610, "height": 852}, "alpha": false, "filters": {"geometry": {"imageSize": {"quotient": 1.0096, "tooLarge": true, "tooSmall": false}, "imageFormat": {"quotient": 0.716, "tooTall": false, "tooWide": false}}, "probability": {"unconfident": false}, "allPassed": false}}], "dataCV": []}