RED-5718: Run the ocr service as specific user

.gitlab-ci.yml

@@ -1,6 +0,0 @@
-variables:
-  SONAR_PROJECT_KEY: 'LM_license-service'
-include:
-  - project: 'gitlab/gitlab'
-    ref: 'main'
-    file: 'ci-templates/maven_java.yml'

bamboo-specs/pom.xml

@@ -0,0 +1,37 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.atlassian.bamboo</groupId>
+        <artifactId>bamboo-specs-parent</artifactId>
+        <version>8.1.3</version>
+        <relativePath/>
+    </parent>
+
+    <artifactId>bamboo-specs</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.atlassian.bamboo</groupId>
+            <artifactId>bamboo-specs-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.atlassian.bamboo</groupId>
+            <artifactId>bamboo-specs</artifactId>
+        </dependency>
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+
+    <!-- run 'mvn test' to perform offline validation of the plan -->
+    <!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
+</project>

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -0,0 +1,125 @@
+package buildjob;
+
+import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
+
+import java.time.LocalTime;
+
+import com.atlassian.bamboo.specs.api.BambooSpec;
+import com.atlassian.bamboo.specs.api.builders.BambooKey;
+import com.atlassian.bamboo.specs.api.builders.Variable;
+import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
+import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
+import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
+import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions;
+import com.atlassian.bamboo.specs.api.builders.plan.Job;
+import com.atlassian.bamboo.specs.api.builders.plan.Plan;
+import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
+import com.atlassian.bamboo.specs.api.builders.plan.Stage;
+import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
+import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
+import com.atlassian.bamboo.specs.api.builders.project.Project;
+import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
+import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
+import com.atlassian.bamboo.specs.builders.task.ScriptTask;
+import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
+import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
+import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
+import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
+import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
+import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
+import com.atlassian.bamboo.specs.util.BambooServer;
+
+/**
+ * Plan configuration for Bamboo.
+ * Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
+ */
+@BambooSpec
+public class PlanSpec {
+
+    private static final String SERVICE_NAME = "ocr-service";
+
+    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
+
+
+    /**
+     * Run main to publish plan on Bamboo
+     */
+    public static void main(final String[] args) throws Exception {
+        //By default credentials are read from the '.credentials' file.
+        BambooServer bambooServer = new BambooServer("http://localhost:8085");
+
+        Plan plan = new PlanSpec().createPlan();
+        bambooServer.publish(plan);
+        PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
+        bambooServer.publish(planPermission);
+
+        Plan secPlan = new PlanSpec().createSecBuild();
+        bambooServer.publish(secPlan);
+        PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
+        bambooServer.publish(secPlanPermission);
+    }
+
+
+    private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
+
+        Permissions permission = new Permissions().userPermissions("atlbamboo",
+                        PermissionType.EDIT,
+                        PermissionType.VIEW,
+                        PermissionType.ADMIN,
+                        PermissionType.CLONE,
+                        PermissionType.BUILD)
+                .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
+                .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
+                .loggedInUserPermissions(PermissionType.VIEW)
+                .anonymousUserPermissionView();
+        return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
+    }
+
+
+    private Project project() {
+
+        return new Project().name("RED").key(new BambooKey("RED"));
+    }
+
+
+    public Plan createPlan() {
+
+        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
+                .variables(new Variable("maven_add_param", ""))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
+                                        .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
+                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
+                                new ScriptTask().description("Build").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh").argument(SERVICE_NAME),
+                                createJUnitParserTask().description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true),
+                                new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL),
+                                new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new BitbucketServerTrigger())
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
+                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
+                        .notificationForCommitters());
+    }
+
+
+    public Plan createSecBuild() {
+
+        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
+                                        .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
+                                new VcsCheckoutTask().description("Checkout Default Repository").checkoutItems(new CheckoutItem().defaultRepository()),
+                                new ScriptTask().description("Sonar").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh").argument(SERVICE_NAME))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
+                                .dockerRunArguments("--net=host")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
+    }
+
+}

bamboo-specs/src/main/resources/scripts/build-java.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+set -e
+
+SERVICE_NAME=$1
+
+if [[ "$bamboo_planRepository_branchName" == "master" ]]
+then
+	echo "building on master branch"
+    branchVersion=$(cat pom.xml | grep -Eo "<version>.*" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|')
+	echo "branch version is : $branchVersion"
+    latestVersion=$(semver $( git tag -l $branchVersion.* ) | tail -n1)
+	echo "latestVersion is : $latestVersion"
+    newVersion="$(semver $latestVersion -p -i minor)"
+	echo "newVersion is : $newVersion"
+elif [[ "$bamboo_planRepository_branchName" == release* ]]
+then
+    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
+    latestVersion=$(semver $( git tag -l $branchVersion.* ) | tail -n1)
+    newVersion="$(semver $latestVersion -p -i patch)"
+elif [[ "${bamboo_version_tag}" != "dev" ]]
+then
+    newVersion="${bamboo_version_tag}"
+else
+    mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        --no-transfer-progress \
+        ${bamboo_maven_add_param} \
+        clean install \
+        -Djava.security.egd=file:/dev/./urandomelse
+    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
+    exit 0
+fi
+
+echo "gitTag=${newVersion}" > git.tag
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    versions:set \
+    -DnewVersion=${newVersion}
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    versions:set  \
+    -DnewVersion=${newVersion}
+
+mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    --no-transfer-progress \
+    clean deploy \
+    ${bamboo_maven_add_param} \
+    -e \
+    -DdeployAtEnd=true \
+    -Dmaven.wagon.http.ssl.insecure=true \
+    -Dmaven.wagon.http.ssl.allowall=true \
+    -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
+    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    package
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    docker:push

bamboo-specs/src/main/resources/scripts/sonar-java.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -e
+
+SERVICE_NAME=$1
+
+echo "build jar binaries"
+mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    --no-transfer-progress \
+    clean install \
+    -Djava.security.egd=file:/dev/./urandomelse
+
+echo "dependency-check:aggregate"
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    org.owasp:dependency-check-maven:aggregate
+
+if [[ -z "${bamboo_repository_pr_key}" ]]
+then
+    echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
+    mvn --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        sonar:sonar \
+        -Dsonar.projectKey=RED_$SERVICE_NAME \
+        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
+        -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
+        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
+        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
+        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
+else
+    echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
+    mvn --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        sonar:sonar \
+        -Dsonar.projectKey=RED_$SERVICE_NAME \
+        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
+        -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
+        -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
+        -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
+        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
+        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
+        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
+fi

bamboo-specs/src/test/java/buildjob/PlanSpecTest.java

@@ -0,0 +1,22 @@
+package buildjob;
+
+import org.junit.Test;
+
+import com.atlassian.bamboo.specs.api.builders.plan.Plan;
+import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException;
+import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
+
+public class PlanSpecTest {
+
+    @Test
+    public void checkYourPlanOffline() throws PropertiesValidationException {
+
+        Plan plan = new PlanSpec().createPlan();
+        EntityPropertiesBuilders.build(plan);
+
+        Plan secPlan = new PlanSpec().createSecBuild();
+        EntityPropertiesBuilders.build(secPlan);
+
+    }
+
+}

ocr-service-image-v1/pom.xml

@@ -3,16 +3,16 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <groupId>com.knecon.fforesight</groupId>
+        <groupId>com.iqser.red</groupId>
         <artifactId>platform-docker-dependency</artifactId>
-        <version>0.1.0</version>
+        <version>1.2.0</version>
         <relativePath/>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>ocr-service-image-v1</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>3.0-SNAPSHOT</version>
+    <version>1.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <properties>

ocr-service-image-v1/src/main/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM red/base-image:2.0.2
+FROM red/base-image:baseImageUser5
 
 COPY "libs/pdftron/OCRModuleLinux.tar.gz" .
 RUN tar xvzf OCRModuleLinux.tar.gz

ocr-service-v1/ocr-service-api-v1/pom.xml

@@ -7,14 +7,27 @@
     <parent>
         <groupId>com.iqser.red.service</groupId>
         <artifactId>ocr-service-v1</artifactId>
-        <version>3.0-SNAPSHOT</version>
+        <version>1.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>ocr-service-api-v1</artifactId>
+    <version>1.0-SNAPSHOT</version>
 
+    <properties>
+        <persistence-service.version>1.269.0</persistence-service.version>
+        <redaction-service.version>3.155.0</redaction-service.version>
+        <dsljson.version>1.9.9</dsljson.version>
+    </properties>
 
     <dependencies>
 
+        <!-- https://mvnrepository.com/artifact/com.dslplatform/dsl-json-java8 -->
+        <dependency>
+            <groupId>com.dslplatform</groupId>
+            <artifactId>dsl-json-java8</artifactId>
+            <version>${dsljson.version}</version>
+        </dependency>
+
         <dependency>
             <!-- This dependency contains annotations that are used in specifying REST endpoints. -->
             <!-- It is optional since not all users of this API might use Feign. -->
@@ -28,6 +41,28 @@
             <artifactId>spring-web</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>persistence-service-api-v1</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>redaction-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
+            <version>${persistence-service.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>redaction-service-api-v1</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>persistence-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
+            <version>${redaction-service.version}</version>
+        </dependency>
 
     </dependencies>
 </project>

ocr-service-v1/ocr-service-api-v1/src/test/java/com/iqser/red/service/ocr/v1/api/model/IdentityTest.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.ocr.v1.api.model;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import org.junit.jupiter.api.Test;
-
-public class IdentityTest {
-
-    @Test
-    public void mockTest() {
-
-        int i = 1;
-        assertThat(i).isEqualTo(1);
-    }
-
-}
-

ocr-service-v1/ocr-service-server-v1/pom.xml

@@ -7,43 +7,23 @@
     <parent>
         <groupId>com.iqser.red.service</groupId>
         <artifactId>ocr-service-v1</artifactId>
-        <version>3.0-SNAPSHOT</version>
+        <version>1.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>ocr-service-server-v1</artifactId>
-
-    <properties>
-        <tennat-commons.version>0.10.0</tennat-commons.version>
-        <persistence-service.version>2.118.0</persistence-service.version>
-        <pdftron-logic-commons.version>2.18.0</pdftron-logic-commons.version>
-    </properties>
+    <version>1.0-SNAPSHOT</version>
 
     <dependencies>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>tenant-commons</artifactId>
-            <version>${tennat-commons.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-internal-api-v1</artifactId>
-            <version>${persistence-service.version}</version>
-        </dependency>
         <dependency>
             <groupId>com.iqser.red.service</groupId>
             <artifactId>ocr-service-api-v1</artifactId>
             <version>${project.version}</version>
         </dependency>
+
         <dependency>
             <groupId>com.iqser.red.commons</groupId>
             <artifactId>storage-commons</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>pdftron-logic-commons</artifactId>
-            <version>${pdftron-logic-commons.version}</version>
-        </dependency>
         <dependency>
             <groupId>com.iqser.red.commons</groupId>
             <artifactId>spring-commons</artifactId>
@@ -59,16 +39,17 @@
         <dependency>
             <groupId>com.pdftron</groupId>
             <artifactId>PDFNet</artifactId>
-            <version>10.1.0</version>
+            <version>9.4.0</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-amqp</artifactId>
+            <version>2.3.1.RELEASE</version>
         </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-kms</artifactId>
-            <version>1.12.440</version>
+            <version>1.12.158</version>
         </dependency>
         <dependency>
             <groupId>com.google.guava</groupId>
@@ -84,6 +65,7 @@
         <dependency>
             <groupId>org.springframework.amqp</groupId>
             <artifactId>spring-rabbit-test</artifactId>
+            <version>2.3.1</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -99,6 +81,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
+            <version>3.12.0</version>
         </dependency>
     </dependencies>
     <build>
@@ -127,6 +110,7 @@
                 <configuration>
                     <annotationProcessors>
                         <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
+                        <annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
                     </annotationProcessors>
                 </configuration>
             </plugin>

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/Application.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.ocr.v1.server;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
-import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -11,22 +10,18 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.annotation.EnableAsync;
 
-import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
-import com.iqser.red.pdftronlogic.commons.WatermarkRemovalService;
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
 
 @EnableAsync
-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
 @EnableConfigurationProperties(OcrServiceSettings.class)
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
-@Import({MessagingConfiguration.class, StorageAutoConfiguration.class})
+@Import({DefaultWebMvcConfiguration.class, MessagingConfiguration.class})
 @EnableFeignClients(basePackageClasses = FileStatusProcessingUpdateClient.class)
 public class Application {
 
@@ -47,18 +42,4 @@ public class Application {
         return new TimedAspect(registry);
     }
 
-
-    @Bean
-    public InvisibleElementRemovalService invisibleElementRemovalService() {
-
-        return new InvisibleElementRemovalService();
-    }
-
-
-    @Bean
-    public WatermarkRemovalService watermarkRemovalService() {
-
-        return new WatermarkRemovalService();
-    }
-
 }

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/client/DossierClient.java

@@ -1,10 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.client;
-
-import org.springframework.cloud.openfeign.FeignClient;
-
-import com.iqser.red.service.persistence.service.v1.api.internal.resources.DossierResource;
-
-@FeignClient(name = "DossierResource", url = "${persistence-service.url}")
-public interface DossierClient extends DossierResource {
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/client/DossierTemplateClient.java

@@ -1,10 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.client;
-
-import org.springframework.cloud.openfeign.FeignClient;
-
-import com.iqser.red.service.persistence.service.v1.api.internal.resources.DossierTemplateResource;
-
-@FeignClient(name = "DossierTemplateResource", url = "${persistence-service.url}")
-public interface DossierTemplateClient extends DossierTemplateResource {
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/client/FileStatusProcessingUpdateClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.ocr.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;
+import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
 
 @FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
 public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/configuration/MessagingConfiguration.java

@@ -20,9 +20,6 @@ public class MessagingConfiguration {
 
     public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue";
 
-    public static final String X_ERROR_INFO_HEADER = "x-error-message";
-    public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
-
 
     @Bean
     public Queue ocrQueue() {

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/initializer/PDFNetInitializer.java

@@ -1,12 +1,14 @@
 package com.iqser.red.service.ocr.v1.server.initializer;
 
+import javax.annotation.PostConstruct;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
 import com.pdftron.pdf.PDFNet;
 
-import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
 
 @Component
 @RequiredArgsConstructor
@@ -27,5 +29,7 @@ public class PDFNetInitializer {
         PDFNet.setTempPath("/tmp/pdftron");
         PDFNet.addResourceSearchPath(ocrModulePath);
         PDFNet.initialize(pdftronLicense);
+
     }
+
 }

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/model/ClippingPathStack.java

@@ -0,0 +1,68 @@
+package com.iqser.red.service.ocr.v1.server.model;
+
+import static com.iqser.red.service.ocr.v1.server.service.InvisibleElementRemovalService.TOLERANCE;
+
+import java.awt.geom.Area;
+import java.awt.geom.GeneralPath;
+import java.awt.geom.Rectangle2D;
+import java.util.Deque;
+import java.util.LinkedList;
+
+import com.pdftron.pdf.Rect;
+
+import lombok.Data;
+import lombok.SneakyThrows;
+
+@Data
+public class ClippingPathStack {
+
+    private Deque<Area> stack = new LinkedList<>();
+
+
+    @SneakyThrows
+    public ClippingPathStack(Rect rectangle) {
+
+        stack.push(new Area(new Rectangle2D.Double(rectangle.getX1(), rectangle.getY1(), rectangle.getWidth(), rectangle.getHeight()).getBounds2D()));
+    }
+
+
+    @SneakyThrows
+    public void intersectClippingPath(GeneralPath path) {
+
+        getCurrentClippingPath().intersect(new Area(path));
+    }
+
+
+    public boolean almostIntersects(double x, double y, double width, double height) {
+        // To address inconsistencies in the calculation of the bounding box we slightly increase the rectangle
+        // Height or width are zero for straight lines, even though they are being rendered. Therefore, height or width must be at minimum >0.
+
+        double x_with_tolerance = x > 0 ? x - TOLERANCE : x + TOLERANCE;
+        double y_with_tolerance = y > 0 ? y - TOLERANCE : y + TOLERANCE;
+        double width_with_tolerance = width + (2 * TOLERANCE);
+        double height_with_tolerance = height + (2 * TOLERANCE);
+        return getCurrentClippingPath().intersects(x_with_tolerance, y_with_tolerance, width_with_tolerance, height_with_tolerance);
+    }
+
+
+    public Area getCurrentClippingPath() {
+
+        return stack.peek();
+    }
+
+
+    public void enterNewGState() {
+
+        Area current = stack.peek();
+        Area cloned = new Area();
+        cloned.add(current);
+        stack.push(cloned);
+    }
+
+
+    public void leaveGState() {
+
+        stack.pop();
+    }
+
+}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/model/ElementFeatures.java

@@ -0,0 +1,170 @@
+package com.iqser.red.service.ocr.v1.server.model;
+
+import static com.iqser.red.service.ocr.v1.server.service.InvisibleElementRemovalService.TOLERANCE;
+
+import java.awt.geom.Rectangle2D;
+
+import com.pdftron.common.PDFNetException;
+import com.pdftron.pdf.Element;
+import com.pdftron.pdf.Rect;
+
+import lombok.AccessLevel;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Getter
+@SuperBuilder
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class ElementFeatures {
+
+    int elementType;
+    Rectangle2D boundingBox;
+
+
+    public boolean almostMatches(Element element) throws PDFNetException {
+
+        return element.getType() == elementType && //
+                element.getBBox() != null && //
+                rectsAlmostMatch(element.getBBox());
+    }
+
+
+    protected boolean almostEqual(double a, double b) {
+
+        return Math.abs(a - b) < TOLERANCE;
+    }
+
+
+    @SneakyThrows
+    private boolean rectsAlmostMatch(Rect bBox) {
+        // To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance
+
+        return almostEqual(bBox.getX1(), boundingBox.getX()) && //
+                almostEqual(bBox.getY1(), boundingBox.getY()) && //
+                almostEqual(bBox.getWidth(), boundingBox.getWidth()) && //
+                almostEqual(bBox.getHeight(), boundingBox.getHeight());
+    }
+
+
+    @EqualsAndHashCode(callSuper = true)
+    @Getter
+    @SuperBuilder
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    private static class Text extends ElementFeatures {
+
+        String text;
+        int font;
+        double fontsize;
+
+
+        @Override
+        public boolean almostMatches(Element element) throws PDFNetException {
+
+            return super.almostMatches(element) && //
+                    text.equals(element.getTextString()) && //
+                    font == element.getGState().getFont().getType() && //
+                    almostEqual(fontsize, element.getGState().getFontSize());
+        }
+
+    }
+
+    @EqualsAndHashCode(callSuper = true)
+    @Getter
+    @SuperBuilder
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    private static class Path extends ElementFeatures {
+
+        boolean isClippingPath;
+        boolean isClipWindingFill;
+        boolean isStroked;
+        boolean isFilled;
+        boolean isWindingFill;
+
+
+        @Override
+        public boolean almostMatches(Element element) throws PDFNetException {
+
+            return super.almostMatches(element) && //
+                    isClippingPath == element.isClippingPath() && //
+                    isClipWindingFill == element.isClipWindingFill() && //
+                    isStroked == element.isStroked() && //
+                    isFilled == element.isFilled() && //
+                    isWindingFill == element.isWindingFill();
+
+        }
+
+    }
+
+    @EqualsAndHashCode(callSuper = true)
+    @Getter
+    @SuperBuilder
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    private static class Image extends ElementFeatures {
+
+        int dataSize;
+        int height;
+        int width;
+        int renderingIntent;
+        int componentNum;
+        int bitsPerComponent;
+
+
+        @Override
+        public boolean almostMatches(Element element) throws PDFNetException {
+
+            return super.almostMatches(element) && //
+                    dataSize == element.getImageDataSize() && //
+                    height == element.getImageHeight() && //
+                    width == element.getImageWidth() && //
+                    renderingIntent == element.getImageRenderingIntent() && //
+                    componentNum == element.getComponentNum() && //
+                    bitsPerComponent == element.getBitsPerComponent();
+        }
+
+    }
+
+
+    public static ElementFeatures extractFeatures(Element element) throws PDFNetException {
+
+        return switch (element.getType()) {
+            case Element.e_path -> Path.builder()
+                    .elementType(element.getType())
+                    .boundingBox(toRectangle2D(element.getBBox()))
+                    .isClippingPath(element.isClippingPath())
+                    .isClipWindingFill(element.isClipWindingFill())
+                    .isStroked(element.isStroked())
+                    .isFilled(element.isFilled())
+                    .isWindingFill(element.isWindingFill())
+                    .build();
+            case Element.e_text -> Text.builder()
+                    .elementType(element.getType())
+                    .boundingBox(toRectangle2D(element.getBBox()))
+                    .text(element.getTextString())
+                    .font(element.getGState().getFont().getType())
+                    .fontsize(element.getGState().getFontSize())
+                    .build();
+            case Element.e_image, Element.e_inline_image -> Image.builder()
+                    .elementType(element.getType())
+                    .boundingBox(toRectangle2D(element.getBBox()))
+                    .dataSize(element.getImageDataSize())
+                    .height(element.getImageHeight())
+                    .width(element.getImageWidth())
+                    .renderingIntent(element.getImageRenderingIntent())
+                    .componentNum(element.getComponentNum())
+                    .bitsPerComponent(element.getBitsPerComponent())
+                    .build();
+            // This technically should never happen, it's a safetynet
+            default -> throw new RuntimeException("Feature Extraction is not supported for PDFTron.Element with type: " + element.getType());
+        };
+    }
+
+
+    private static Rectangle2D toRectangle2D(Rect rect) throws PDFNetException {
+
+        return new Rectangle2D.Double(rect.getX1(), rect.getY1(), rect.getWidth(), rect.getHeight());
+    }
+
+}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/model/ImagePosition.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.ocr.v1.server.model;
 
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/model/image/ImageServiceResponse.java

@@ -3,12 +3,15 @@ package com.iqser.red.service.ocr.v1.server.model.image;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.dslplatform.json.CompiledJson;
+import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import lombok.Data;
 
 @Data
+@CompiledJson
 public class ImageServiceResponse {
 
     private String dossierId;
@@ -16,11 +19,13 @@ public class ImageServiceResponse {
 
     @JsonProperty(value = "imageMetadata")
     @JsonAlias("data")
+    @JsonAttribute(alternativeNames = {"imageMetadata"})
     private List<ImageMetadata> data = new ArrayList<>();
 
 
     @JsonProperty(value = "imageMetadata")
     @JsonAlias("data")
+    @JsonAttribute(alternativeNames = {"imageMetadata"})
     public void setData(List<ImageMetadata> data) {this.data = data;}
 
 }

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/DossierService.java

@@ -1,36 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.service;
-
-import org.springframework.http.HttpStatus;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.ocr.v1.server.client.DossierClient;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
-
-import feign.FeignException;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
-public class DossierService {
-
-    DossierClient dossierClient;
-
-    public Dossier getDossier(String dossierId) {
-
-        try {
-            return dossierClient.getDossierById(dossierId, true, false);
-        } catch (FeignException e) {
-            if (e.status() == HttpStatus.NOT_FOUND.value()) {
-                return null;
-            } else {
-                throw e;
-            }
-        }
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/DossierTemplateService.java

@@ -1,36 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.service;
-
-import org.springframework.http.HttpStatus;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.ocr.v1.server.client.DossierTemplateClient;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
-
-import feign.FeignException;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
-public class DossierTemplateService {
-
-    DossierTemplateClient dossierTemplateClient;
-
-    public DossierTemplate getDossierTemplate(String dossierTemplateId) {
-
-        try {
-            return dossierTemplateClient.getDossierTemplateById(dossierTemplateId);
-        } catch (FeignException e) {
-            if (e.status() == HttpStatus.NOT_FOUND.value()) {
-                return null;
-            } else {
-                throw e;
-            }
-        }
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/FileStorageService.java

@@ -7,9 +7,8 @@ import org.apache.commons.io.IOUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.ocr.v1.server.model.image.ImageServiceResponse;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -32,39 +31,39 @@ public class FileStorageService {
     @SneakyThrows
     public byte[] getOriginalFile(String dossierId, String fileId) {
 
-        return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
+        return IOUtils.toByteArray(storageService.getObject(getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
     }
 
 
     @SneakyThrows
     public InputStream getOriginalFileAsStream(String dossierId, String fileId) {
 
-        return storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
+        return storageService.getObject(getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
     }
 
 
     public void storeOriginalFile(String dossierId, String fileId, InputStream stream) {
 
-        storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
+        storageService.storeObject(getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
     }
 
 
     public boolean untouchedFileExists(String dossierId, String fileId) {
 
-        return storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED));
+        return storageService.objectExists(getStorageId(dossierId, fileId, FileType.UNTOUCHED));
     }
 
 
     public void storeUntouchedFile(String dossierId, String fileId, byte[] data) {
 
-        storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
+        storageService.storeObject(getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
     }
 
 
     @SneakyThrows
     public ImageServiceResponse getImageServiceResponse(String dossierId, String fileId) {
 
-        return storageService.readJSONObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
+        return storageService.readJSONObject(getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
     }
 
 }

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/ImagePositionRetrievalService.java

@@ -47,17 +47,16 @@ public class ImagePositionRetrievalService {
         ElementReader reader = new ElementReader();
         for (int pageId = 1; pageId <= pdfDoc.getPageCount(); ++pageId) {
             RectCollection imagePositions = new RectCollection();
-
             reader.begin(pdfDoc.getPage(pageId));
+
             findImagePositionsOnPage(reader, imagePositions, pdfDoc.getPage(pageId), mirrorY);
             imagePositions = mergeOverlappingRects(imagePositions);
-            reader.end();
 
+            reader.end();
             if (imagePositions.getNumRects() > 0) {
                 pageIdToImagePositions.put(pageId, imagePositions);
             }
         }
-        reader.destroy();
         return pageIdToImagePositions;
     }
 

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/InvisibleElementRemovalService.java

@@ -0,0 +1,450 @@
+package com.iqser.red.service.ocr.v1.server.service;
+
+import java.awt.Shape;
+import java.awt.geom.AffineTransform;
+import java.awt.geom.GeneralPath;
+import java.awt.geom.Rectangle2D;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.springframework.stereotype.Service;
+
+import com.google.common.primitives.Bytes;
+import com.google.common.primitives.Doubles;
+import com.iqser.red.service.ocr.v1.server.model.ClippingPathStack;
+import com.iqser.red.service.ocr.v1.server.model.ElementFeatures;
+import com.pdftron.common.Matrix2D;
+import com.pdftron.common.PDFNetException;
+import com.pdftron.pdf.ColorPt;
+import com.pdftron.pdf.ColorSpace;
+import com.pdftron.pdf.Element;
+import com.pdftron.pdf.ElementBuilder;
+import com.pdftron.pdf.ElementReader;
+import com.pdftron.pdf.ElementWriter;
+import com.pdftron.pdf.GState;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.pdf.PageIterator;
+import com.pdftron.pdf.PathData;
+import com.pdftron.pdf.Rect;
+import com.pdftron.sdf.Obj;
+import com.pdftron.sdf.SDFDoc;
+
+import lombok.Builder;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class InvisibleElementRemovalService {
+
+    static public final double TOLERANCE = 1e-3;
+
+
+    /**
+     * Removes all hidden Text, Path and Image Elements from a PDF Document.
+     * handled cases:
+     * -Text which is transparent or is set to not render
+     * -Elements outside of clipping path
+     * -Elements that have been painted over by visible and filled Paths
+     * unhandled cases:
+     * -Elements covered by widely stroked path
+     * -Elements with the same color as background
+     * -Any Text set to clipping with its many interactions with other elements
+     *
+     * @param pdfFile The PDF file to process
+     * @param delta   If this flag is set only the removed Elements will be written to the output file.
+     *                The Elements are red if they are removed by clipping path, blue for transparency, and a green bounding box for overlap.
+     * @param out     OutputStream to write the resulting file to
+     **/
+    @SneakyThrows
+    public void removeInvisibleElements(InputStream pdfFile, OutputStream out, boolean delta) {
+
+        PDFDoc pdfDoc = new PDFDoc(pdfFile);
+
+        ElementWriter writer = new ElementWriter();
+        ElementReader reader = new ElementReader();
+        Set<Long> visitedXObjIds = new TreeSet<>();
+
+        for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
+
+            Page page = iterator.next();
+
+            visitedXObjIds.add(page.getSDFObj().getObjNum());
+
+
+            InvisibleElementRemovalContext context = InvisibleElementRemovalContext.builder()
+                    .reader(reader)
+                    .clippingPathStack(new ClippingPathStack(page.getMediaBox()))
+                    .delta(delta)
+                    .overlappedElements(new ArrayList<>())
+                    .visibleElements(new ArrayList<>())
+                    .visitedXObjIds(visitedXObjIds)
+                    .build();
+
+            removeClippedElementsAndInvisibleTextAndRememberOverlappedElements(page, writer, context);
+
+            context.visitedXObjIds().clear();
+
+            removeOverlappedElements(page, writer, context);
+        }
+        pdfDoc.save(out, SDFDoc.SaveMode.REMOVE_UNUSED, null);
+    }
+
+
+    private void removeClippedElementsAndInvisibleTextAndRememberOverlappedElements(Page page,
+                                                                                    ElementWriter writer,
+                                                                                    InvisibleElementRemovalContext context) throws PDFNetException {
+
+        context.reader().begin(page);
+        writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
+        processElements(writer, context);
+        writer.end();
+        context.reader().end();
+    }
+
+
+    private void processElements(ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
+
+        for (Element element = context.reader().next(); element != null; element = context.reader().next())
+            switch (element.getType()) {
+                case Element.e_image, Element.e_inline_image -> processImages(element, writer, context);
+                case Element.e_text -> processText(element, writer, context);
+                case Element.e_path -> processPath(element, writer, context);
+                case Element.e_form -> processForm(element, writer, context);
+                case Element.e_group_begin -> {
+                    context.clippingPathStack().enterNewGState();
+                    writer.writeElement(element);
+                }
+                case Element.e_group_end -> {
+                    context.clippingPathStack().leaveGState();
+                    writer.writeElement(element);
+                }
+                default -> writer.writeElement(element);
+            }
+    }
+
+
+    private void processImages(Element imageElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
+
+        Rect rect = imageElement.getBBox();
+
+        if (rect == null) {
+            return;
+        }
+
+        boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX1(), rect.getY1(), rect.getWidth(), rect.getHeight());
+
+        if (!context.delta() && inClippingPath) {
+            context.visibleElements().add(ElementFeatures.extractFeatures(imageElement));
+        }
+
+        if (context.delta() ^ inClippingPath) {
+            writer.writeElement(imageElement);
+        }
+    }
+
+
+    private void processText(Element textElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
+
+        Rect rect = textElement.getBBox();
+
+        if (rect == null) {
+            writer.writeElement(textElement);
+            return;
+        }
+
+        GState gState = textElement.getGState();
+
+        boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX1(), rect.getY1(), rect.getWidth(), rect.getHeight());
+
+        boolean isTextVisible = isTextRenderedVisibly(gState);
+
+        if (inClippingPath && isTextVisible) {
+            context.visibleElements().add(ElementFeatures.extractFeatures(textElement));
+        }
+        if (!context.delta()) {
+            if (inClippingPath && isTextVisible) {
+                writer.writeElement(textElement);
+            } else if (textElement.hasTextMatrix()) {
+                /*
+                PDFTron Element with type "text" refers to a Tj command. If a Tm command is just above it in the pdf file, PDFTron will join the two commands and treat them as one Element.
+                hasTextMatrix() checks for this case specifically. Also, Tm changes the position for a whole BT/ET segment, possibly containing multiple Tj commands.
+                Therefore, the position of a following Tj is affected by not writing the first Element.
+                This is why, we write only the Tm command:
+                */
+                writer.writeGStateChanges(textElement);
+            }
+        } else {
+            if (!inClippingPath) {
+                gState.setFillColorSpace(ColorSpace.createDeviceRGB());
+                // red for elements removed by clipping path
+                gState.setFillColor(new ColorPt(1, 0, 0));
+                writer.writeElement(textElement);
+            }
+            if (!isTextVisible) {
+                gState.setFillColorSpace(ColorSpace.createDeviceRGB());
+                // blue for elements removed due to transparency or not rendered
+                gState.setFillColor(new ColorPt(0, 0, 1));
+                gState.setTextRenderMode(GState.e_fill_text);
+                gState.setFillOpacity(1);
+                writer.writeElement(textElement);
+            }
+        }
+    }
+
+
+    private void processForm(Element formElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
+
+        writer.writeElement(formElement);
+        Obj formObj = formElement.getXObject();
+
+        if (!context.visitedXObjIds().contains(formObj.getObjNum())) {
+            context.visitedXObjIds().add(formObj.getObjNum());
+            // writer needs to be newly initialized when entering a new content stream
+            // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
+            ElementWriter formWriter = new ElementWriter();
+            context.reader().formBegin();
+            formWriter.begin(formObj);
+
+            context.reader().clearChangeList();
+            formWriter.setDefaultGState(context.reader());
+
+            processElements(formWriter, context);
+            formWriter.end();
+            context.reader().end();
+        }
+    }
+
+
+    private void processPath(Element pathElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
+        PathData pathData = pathElement.getPathData();
+
+        if (pathData.getOperators().length == 0 && pathData.getPoints().length == 0) {
+            writer.writeGStateChanges(pathElement);
+            return;
+        }
+
+        GeneralPath linePath = convertToGeneralPath(pathData);
+
+        //transform path to initial user space
+        var ctm = pathElement.getCTM();
+        var affineTransform = toAffineTransform(ctm);
+        linePath.transform(affineTransform);
+
+        var rect = linePath.getBounds2D();
+
+        boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX(), rect.getY(), rect.getWidth(), rect.getHeight());
+
+        if (pathElement.isClippingPath()) {
+            if (pathElement.isClipWindingFill()) {
+                linePath.setWindingRule(GeneralPath.WIND_NON_ZERO);
+            } else {
+                linePath.setWindingRule(GeneralPath.WIND_EVEN_ODD);
+            }
+
+            context.clippingPathStack().intersectClippingPath(linePath);
+            pathElement.setPathClip(!context.delta());
+            writer.writeElement(pathElement);
+
+        } else {
+            if (pathElement.isWindingFill()) {
+                linePath.setWindingRule(GeneralPath.WIND_NON_ZERO);
+            } else {
+                linePath.setWindingRule(GeneralPath.WIND_EVEN_ODD);
+            }
+
+            if (inClippingPath) {
+                if (isFilledAndNonTransparent(pathElement)) {
+                    List<ElementFeatures> currentOverlappedElements = context.visibleElements()
+                            .stream()
+                            .filter(features -> almostContains(linePath, features.getBoundingBox()))
+                            .toList();
+                    context.overlappedElements().addAll(currentOverlappedElements);
+                    context.visibleElements().removeAll(currentOverlappedElements);
+                }
+                context.visibleElements().add(ElementFeatures.extractFeatures(pathElement));
+                if (!context.delta()) {
+                    writer.writeElement(pathElement);
+                }
+            }
+            if (context.delta() && !inClippingPath) {
+                pathElement.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
+                pathElement.getGState().setFillColor(new ColorPt(1, 0, 0));
+                pathElement.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+                pathElement.getGState().setStrokeColor(new ColorPt(1, 0, 0));
+                writer.writeElement(pathElement);
+            }
+        }
+    }
+
+
+    private void removeOverlappedElements(Page page, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
+
+        context.reader().begin(page);
+        writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
+        if (context.delta()) {
+            // green for element removed due to overlapping
+            context.overlappedElements().forEach(feature -> drawBBox(writer, feature.getBoundingBox(), "#00FF00"));
+            context.overlappedElements().clear();
+        }
+        processOverlappedElements(writer, context);
+        writer.end();
+        context.reader().end();
+
+        if (context.overlappedElements().size() > 0) {
+            log.warn(context.overlappedElements().size() + " overlapped elements have not been found or removed");
+        }
+    }
+
+
+    private void processOverlappedElements(ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
+
+        for (Element element = context.reader().next(); element != null; element = context.reader().next()) {
+            switch (element.getType()) {
+                case Element.e_form -> processFormOverlappedElements(writer, element, context);
+                case Element.e_path, Element.e_image, Element.e_inline_image, Element.e_text -> {
+                    boolean anyMatch = false;
+                    for (ElementFeatures elementToRemove : context.overlappedElements()) {
+                        if (elementToRemove.almostMatches(element)) {
+                            context.overlappedElements().remove(elementToRemove);
+                            anyMatch = true;
+                            break;
+                        }
+                    }
+                    if (!anyMatch) {
+                        writer.writeElement(element);
+                    } else if (element.getType() == 3 && element.hasTextMatrix()) {
+                        /*
+                        PDFTron Element with type "text" refers to a Tj command. If a Tm command is just above it in the pdf file, PDFTron will join the two commands and treat them as one Element.
+                        hasTextMatrix() checks for this case specifically. Also, Tm changes the position for a whole BT/ET segment, possibly containing multiple Tj commands.
+                        Therefore, the position of a following Tj is affected by not writing the first Element.
+                        This is why, we write only the Tm command:
+                        */
+                        writer.writeGStateChanges(element);
+                    }
+                }
+                default -> writer.writeElement(element);
+            }
+        }
+    }
+
+
+    private void processFormOverlappedElements(ElementWriter writer, Element formElement, InvisibleElementRemovalContext context) throws PDFNetException {
+
+        writer.writeElement(formElement);
+        Obj formObj = formElement.getXObject();
+
+        if (!context.visitedXObjIds().contains(formObj.getObjNum())) {
+            context.visitedXObjIds().add(formObj.getObjNum());
+            // writer needs to be newly initialized when entering a new content stream
+            // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
+            ElementWriter formWriter = new ElementWriter();
+            context.reader().formBegin();
+            formWriter.begin(formObj);
+
+            context.reader().clearChangeList();
+            formWriter.setDefaultGState(context.reader());
+
+            processOverlappedElements(formWriter, context);
+            formWriter.end();
+            context.reader().end();
+        }
+    }
+
+
+    private boolean isTextRenderedVisibly(GState gState) throws PDFNetException {
+
+        return gState.getTextRenderMode() != GState.e_invisible_text && //
+                !(gState.getTextRenderMode() == GState.e_fill_text && gState.getFillOpacity() == 0) && //
+                !(gState.getTextRenderMode() == GState.e_stroke_text && gState.getStrokeOpacity() == 0) && //
+                !(gState.getTextRenderMode() == GState.e_fill_stroke_text && gState.getFillOpacity() == 0 && gState.getStrokeOpacity() == 0);
+    }
+
+
+    private GeneralPath convertToGeneralPath(PathData pathData) throws PDFNetException {
+
+        GeneralPath linePath = new GeneralPath();
+        Iterator<Double> points = Doubles.asList(pathData.getPoints()).iterator();
+        Iterable<Byte> operators = Bytes.asList(pathData.getOperators());
+        for (var operator : operators) {
+            switch (operator) {
+                case PathData.e_moveto -> linePath.moveTo(points.next(), points.next());
+                case PathData.e_lineto -> linePath.lineTo(points.next(), points.next());
+                case PathData.e_cubicto -> linePath.curveTo(points.next(), points.next(), points.next(), points.next(), points.next(), points.next());
+                case PathData.e_closepath -> linePath.closePath();
+                case PathData.e_rect -> {
+                    double x = points.next();
+                    double y = points.next();
+                    double w = points.next();
+                    double h = points.next();
+                    linePath.moveTo(x, y);
+                    linePath.lineTo(x + w, y);
+                    linePath.lineTo(x + w, y + h);
+                    linePath.lineTo(x, y + h);
+                    linePath.closePath();
+                }
+                default -> throw new PDFNetException("Invalid Element Type", 0, "", "", "");
+            }
+        }
+        return linePath;
+    }
+
+
+    private boolean almostContains(Shape outer, Rectangle2D inner) {
+        //To address inconsistencies in the calculation of the bounding box we slightly shrink the inner rectangle
+
+        double x_with_tolerance = inner.getX() >= 0 ? inner.getX() + TOLERANCE : inner.getX() - TOLERANCE;
+        double y_with_tolerance = inner.getY() >= 0 ? inner.getY() + TOLERANCE : inner.getY() - TOLERANCE;
+        double height_with_tolerance = inner.getHeight() - (2 * TOLERANCE);
+        double width_with_tolerance = inner.getWidth() - (2 * TOLERANCE);
+        Rectangle2D innerRect = new Rectangle2D.Double(x_with_tolerance, y_with_tolerance, width_with_tolerance, height_with_tolerance);
+
+        return outer.contains(innerRect);
+    }
+
+
+    private boolean isFilledAndNonTransparent(Element element) throws PDFNetException {
+
+        return element.isFilled() && element.getGState().getFillOpacity() == 1;
+    }
+
+
+    @SneakyThrows
+    private void drawBBox(ElementWriter writer, Rectangle2D r, String hexcolor) {
+
+        ColorPt colorPt = new ColorPt(Integer.valueOf(hexcolor.substring(1, 3), 16) / 255d,
+                Integer.valueOf(hexcolor.substring(3, 5), 16) / 255d,
+                Integer.valueOf(hexcolor.substring(5, 7), 16) / 255d);
+        ElementBuilder eb = new ElementBuilder();
+        Element rect = eb.createRect(r.getX(), r.getY(), r.getWidth(), r.getHeight());
+        rect.setPathStroke(true);
+        rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+        rect.getGState().setStrokeColor(colorPt);
+        writer.writePlacedElement(rect);
+    }
+
+
+    private static AffineTransform toAffineTransform(Matrix2D ctm) throws PDFNetException {
+
+        return new AffineTransform(ctm.getA(), ctm.getB(), ctm.getC(), ctm.getD(), ctm.getH(), ctm.getV());
+    }
+
+
+    @Builder
+    private record InvisibleElementRemovalContext(
+            boolean delta,
+            ElementReader reader,
+            ClippingPathStack clippingPathStack,
+            List<ElementFeatures> overlappedElements,
+            List<ElementFeatures> visibleElements,
+            Set<Long> visitedXObjIds) {
+
+    }
+
+}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/OCRService.java

@@ -7,19 +7,15 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.HashMap;
 import java.util.Map;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
-import com.iqser.red.pdftronlogic.commons.WatermarkRemovalService;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.ocr.v1.api.model.OCRStatusUpdateResponse;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.pdftron.common.PDFNetException;
 import com.pdftron.pdf.OCRModule;
 import com.pdftron.pdf.OCROptions;
@@ -46,16 +42,12 @@ public class OCRService {
 
     private final RabbitTemplate rabbitTemplate;
 
-    private final WatermarkRemovalService watermarkRemovalService;
+    private final ObjectMapper objectMapper;
 
     private final InvisibleElementRemovalService invisibleElementRemovalService;
 
     private final ImagePositionRetrievalService imagePositionRetrievalService;
 
-    private final DossierService dossierService;
-
-    private final DossierTemplateService dossierTemplateService;
-
 
     /**
      * First loads the PDF Document from storage.
@@ -73,33 +65,20 @@ public class OCRService {
     @Timed("redactmanager_runOcrOnDocument")
     public void runOcrOnDocument(String dossierId, String fileId, OutputStream out) throws IOException {
 
-        ByteArrayOutputStream transferOutputStream = new ByteArrayOutputStream();
-        InputStream fileStream = fileStorageService.getOriginalFileAsStream(dossierId, fileId);
-
-        try {
-            Dossier dossier = dossierService.getDossier(dossierId);
-            DossierTemplate dossierTemplate = dossierTemplateService.getDossierTemplate(dossier.getDossierTemplateId());
-
-            if (dossierTemplate.isRemoveWatermark()) {
-                watermarkRemovalService.removeWatermarks(fileStream, transferOutputStream);
-                fileStream.close();
-                fileStream = new ByteArrayInputStream(transferOutputStream.toByteArray());
-                transferOutputStream.close();
-                transferOutputStream = new ByteArrayOutputStream();
+        try (ByteArrayOutputStream transferOutputStream = new ByteArrayOutputStream()) {
+            try (InputStream fileStream = fileStorageService.getOriginalFileAsStream(dossierId, fileId)) {
+                long start = System.currentTimeMillis();
+                log.debug("Start invisible element removal for file with dossierId {} and fileId {}", dossierId, fileId);
+                invisibleElementRemovalService.removeInvisibleElements(fileStream, transferOutputStream, false);
+                long end = System.currentTimeMillis();
+                log.debug("Invisible element removal successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, format("%.1f", (end - start) / 1000.0));
             }
-
-            invisibleElementRemovalService.removeInvisibleElements(fileStream, transferOutputStream, false);
-
             try (InputStream transferInputStream = new ByteArrayInputStream(transferOutputStream.toByteArray())) {
-                long ocrStart = System.currentTimeMillis();
+                long start = System.currentTimeMillis();
                 runOcr(transferInputStream, out, fileId);
-                long ocrEnd = System.currentTimeMillis();
-                log.info("ocr successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, format("%.1f", (ocrEnd - ocrStart) / 1000.0));
+                long end = System.currentTimeMillis();
+                log.info("ocr successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, format("%.1f", (end - start) / 1000.0));
             }
-
-        } finally {
-            fileStream.close();
-            transferOutputStream.close();
         }
     }
 
@@ -110,22 +89,18 @@ public class OCRService {
         PDFDoc pdfDoc = new PDFDoc(fileStream);
 
         Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, true);
-        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
-                OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).build());
 
         // Optimization:
         // When a page does not have a TextZone, PDFTron whites out the page. But, PDFTron scans it anyway, resulting in a longer runtime.
         // So, we need to remove pages without images.
         // Furthermore, creating a new document is *much* faster than reusing the same document and adding/removing pages one by one.
-        // Therefore, we create a new Document with a single page for every page that contains an image.
-        // For some reason, if we insert the OCR singlePageDoc into the original PDFDoc inside the loop, for some documents, the RAM Usage increases exponentially with every page.
-        // This is why, we replace the OCRed Pages outside the main loop.
+        // Therefore, we create a new Document with a single page for every page that contains text.
         int numProcessedPages = 0;
-        Map<Integer, PDFDoc> pageIdToSinglePagePdfDoc = new HashMap<>();
         for (Integer pageId : pageIdToRectCollection.keySet()) {
             try {
                 PDFDoc singlePagePdfDoc = extractSinglePagePdfDoc(pdfDoc, pageId);
                 processOcr(pageIdToRectCollection, pageId, singlePagePdfDoc);
+                ++numProcessedPages;
 
                 log.info("{}/{} Page {} done, OCR regions {}",
                         numProcessedPages,
@@ -133,31 +108,37 @@ public class OCRService {
                         pageId,
                         getAllOcrTextZonesAsString(pageIdToRectCollection, pageId));
 
-                pageIdToSinglePagePdfDoc.put(pageId, singlePagePdfDoc);
-                ++numProcessedPages;
+                replaceOriginalPageWithOcrPage(pdfDoc, pageId, singlePagePdfDoc);
+                singlePagePdfDoc.close();
 
                 rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
-                        OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).numberOfOCRedPages(numProcessedPages).build());
+                        objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder()
+                                .fileId(fileId)
+                                .numberOfPagesToOCR(pageIdToRectCollection.size())
+                                .numberOfOCRedPages(numProcessedPages)
+                                .build()));
 
             } catch (PDFNetException e) {
                 log.error("Failed to process page {}", pageId);
                 throw new RuntimeException(e);
             }
         }
-        log.info("Copying {} OCRed Pages into original Document", pageIdToSinglePagePdfDoc.size());
-        pageIdToSinglePagePdfDoc.forEach((pageId, singlePagePdfDoc) -> replaceOriginalPageWithOcrPage(pdfDoc, pageId, singlePagePdfDoc));
-        Optimizer.optimize(pdfDoc);
 
         rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
-                OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).numberOfOCRedPages(numProcessedPages).ocrFinished(true).build());
+                objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder()
+                        .fileId(fileId)
+                        .numberOfPagesToOCR(pageIdToRectCollection.size())
+                        .numberOfOCRedPages(numProcessedPages)
+                        .ocrFinished(true)
+                        .build()));
 
+        Optimizer.optimize(pdfDoc);
         try {
             pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
         } catch (Exception e) {
             log.error("Processed File with fileId {} could not be saved", fileId);
             throw new RuntimeException(e);
         }
-        pdfDoc.close();
     }
 
 
@@ -182,13 +163,11 @@ public class OCRService {
     }
 
 
-    @SneakyThrows
-    private static void replaceOriginalPageWithOcrPage(PDFDoc pdfDoc, Integer pageId, PDFDoc singlePagePdfDoc) {
+    private static void replaceOriginalPageWithOcrPage(PDFDoc pdfDoc, Integer pageId, PDFDoc ocrPageDoc) throws PDFNetException {
 
-        Page ocrPage = singlePagePdfDoc.getPage(1);
+        Page ocrPage = ocrPageDoc.getPage(1);
         pdfDoc.pageInsert(pdfDoc.getPageIterator(pageId), ocrPage);
         pdfDoc.pageRemove(pdfDoc.getPageIterator(pageId + 1));
-        singlePagePdfDoc.close();
     }
 
 

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/OcrMessageReceiver.java

@@ -1,26 +1,20 @@
 package com.iqser.red.service.ocr.v1.server.service;
 
-import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
-import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.time.OffsetDateTime;
-import java.time.temporal.ChronoUnit;
 
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
-import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Service;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.ocr.v1.api.model.DocumentRequest;
 import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 
 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
@@ -31,60 +25,50 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class OcrMessageReceiver {
 
-    private final FileStorageService fileStorageService;
     private final ObjectMapper objectMapper;
+    private final FileStorageService fileStorageService;
     private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
+
     private final OCRService ocrService;
 
 
     @RabbitHandler
     @RabbitListener(queues = MessagingConfiguration.OCR_QUEUE, concurrency = "1")
-    public void receiveOcr(Message in) throws IOException {
+    public void receiveOcr(String in) throws JsonProcessingException {
 
-        DocumentRequest ocrRequestMessage = objectMapper.readValue(in.getBody(), DocumentRequest.class);
+        DocumentRequest ocrRequestMessage = objectMapper.readValue(in, DocumentRequest.class);
 
         log.info("Start ocr for file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
 
-        try {
-            setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+        setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
 
-            if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) {
-                byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
-                fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile);
-            }
-
-            try (var transferStream = new ByteArrayOutputStream()) {
-                ocrService.runOcrOnDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), transferStream);
-                try (var inputStream = new ByteArrayInputStream(transferStream.toByteArray())) {
-                    fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), inputStream);
-                }
-            } catch (IOException e) {
-                log.error("Failed to store file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
-                throw new RuntimeException(e);
-            }
-
-            fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
-        } catch (Exception e) {
-            log.warn("An exception occurred in ocr file stage: {}", e.getMessage());
-            in.getMessageProperties().getHeaders().put(X_ERROR_INFO_HEADER, e.getMessage());
-            in.getMessageProperties().getHeaders().put(X_ERROR_INFO_TIMESTAMP_HEADER, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-            throw e;
+        if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) {
+            byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+            fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile);
         }
+
+        try (var transferStream = new ByteArrayOutputStream()) {
+            ocrService.runOcrOnDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), transferStream);
+            try (var inputStream = new ByteArrayInputStream(transferStream.toByteArray())) {
+                fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), inputStream);
+            }
+        } catch (IOException e) {
+            log.error("Failed to store file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+            throw new RuntimeException(e);
+        }
+
+        fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+
     }
 
 
     @RabbitHandler
     @RabbitListener(queues = MessagingConfiguration.OCR_DLQ, concurrency = "1")
-    public void receiveOcrDQL(Message failedMessage) throws IOException {
+    public void receiveOcrDQL(String in) throws JsonProcessingException {
 
-        DocumentRequest ocrRequestMessage = objectMapper.readValue(failedMessage.getBody(), DocumentRequest.class);
+        DocumentRequest ocrRequestMessage = objectMapper.readValue(in, DocumentRequest.class);
         log.info("OCR DQL received: {}", ocrRequestMessage);
-        String errorMessage = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
-        OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
-        timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
-        fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(),
-                ocrRequestMessage.getFileId(),
-                new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp));
+        fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
     }
 
 

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/settings/OcrServiceSettings.java

@@ -9,5 +9,6 @@ import lombok.Data;
 public class OcrServiceSettings {
 
     private int ocrDPI = 300;
+    private boolean removeWatermark;
 
 }

ocr-service-v1/ocr-service-server-v1/src/main/resources/application-dev.yaml

@@ -1,7 +1,10 @@
 server:
-  port: 8097
+  port: 8086
 
-persistence-service.url: "http://localhost:8085"
-tenant-user-management-service.url: "http://localhost:8091/internal"
+persistence-service.url: "http://persistence-service-v1:8080"
 
-pdftron.license: demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a
+storage:
+  bucket-name: 'redaction'
+  endpoint: 'http://localhost:9000'
+  key: minioadmin
+  secret: minioadmin

ocr-service-v1/ocr-service-server-v1/src/main/resources/application.yml

@@ -2,8 +2,6 @@ info:
   description: OCR Service V1 Server
 
 persistence-service.url: "http://persistence-service-v1:8080"
-tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
-fforesight.tenants.remote: true
 
 server:
   port: 8080
@@ -28,6 +26,9 @@ spring:
           max-interval: 15000
         prefetch: 1
 
+platform.multi-tenancy:
+  enabled: false
+
 
 management:
   endpoint:
@@ -39,6 +40,10 @@ management:
 
 
 storage:
+  signer-type: 'AWSS3V4SignerType'
+  bucket-name: 'redaction'
+  region: 'us-east-1'
+  endpoint: 'https://s3.amazonaws.com'
   backend: 's3'
 
 pdftron.license: ${PDFTRON_LICENSE}

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/AbstractTest.java

@@ -1,117 +0,0 @@
-package com.iqser.red.service.ocr.v1.server;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.MockitoAnnotations;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
-import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
-import com.iqser.red.service.ocr.v1.server.initializer.PDFNetInitializer;
-import com.iqser.red.service.ocr.v1.server.service.DossierService;
-import com.iqser.red.service.ocr.v1.server.service.DossierTemplateService;
-import com.iqser.red.service.ocr.v1.server.utils.FileSystemBackedStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.tenantcommons.TenantsClient;
-import com.pdftron.pdf.PDFNet;
-
-import jakarta.annotation.PostConstruct;
-import lombok.SneakyThrows;
-
-@ExtendWith({SpringExtension.class, MockitoExtension.class})
-@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import({AbstractTest.TestConfiguration.class, PDFNetInitializer.class})
-@AutoConfigureObservability
-public class AbstractTest {
-
-    protected static final String TEST_DOSSIER_TEMPLATE_ID = "1337";
-    protected static final String TEST_DOSSIER_ID = "42";
-
-    @MockBean
-    private TenantsClient tenantsClient;
-    @MockBean
-    protected DossierService dossierService;
-    @MockBean
-    protected DossierTemplateService dossierTemplateService;
-
-    @Autowired
-    protected StorageService storageService;
-
-    @MockBean
-    protected RabbitTemplate rabbitTemplate;
-
-    private static String pdftronLicense;
-
-
-    @BeforeEach
-    public void openMocks() {
-
-        MockitoAnnotations.openMocks(this);
-    }
-
-
-    @PostConstruct
-    @SneakyThrows
-    public void init() {
-
-        PDFNet.initialize(pdftronLicense);
-    }
-
-
-    @AfterAll
-    public static void terminatePDFNet() {
-
-        PDFNet.terminate();
-        System.out.println("PDFNet Terminated");
-    }
-
-
-    @SneakyThrows
-    public void dummyTest() {
-
-        // Build needs one test to not fail.
-        assertThat(1).isEqualTo(1);
-    }
-
-
-    @AfterEach
-    public void cleanupStorage() {
-
-        if (this.storageService instanceof FileSystemBackedStorageService) {
-            ((FileSystemBackedStorageService) this.storageService).clearStorage();
-        }
-    }
-
-
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class TestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inMemoryStorage() {
-
-            return new FileSystemBackedStorageService();
-        }
-
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/OcrServiceIntegrationTest.java

@@ -1,41 +1,70 @@
 package com.iqser.red.service.ocr.v1.server;
 
-import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.when;
 
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.ocr.v1.server.service.FileStorageService;
 import com.iqser.red.service.ocr.v1.server.service.OCRService;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.knecon.fforesight.tenantcommons.TenantContext;
+import com.iqser.red.service.ocr.v1.server.utils.FileSystemBackedStorageService;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.pdftron.common.PDFNetException;
 import com.pdftron.pdf.OCRModule;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.pdf.PageIterator;
+import com.pdftron.pdf.TextExtractor;
 
 import io.micrometer.prometheus.PrometheusMeterRegistry;
 import io.micrometer.prometheus.PrometheusTimer;
 import lombok.SneakyThrows;
 
-@Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
-@SpringBootTest(properties = {"pdftron.ocrmodule.path=/YourOCRModulePath"})
-public class OcrServiceIntegrationTest extends AbstractTest {
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT //
+        , properties = {"pdftron.ocrmodule.path=/YourOCRModulePath"})
+@Import(OcrServiceIntegrationTest.TestConfiguration.class)
+public class OcrServiceIntegrationTest {
+
+    @Autowired
+    protected StorageService storageService;
+
+    @Autowired
+    protected FileStorageService fileStorageService;
 
     @Autowired
     protected ObjectMapper objectMapper;
 
+    @MockBean
+    protected RabbitTemplate rabbitTemplate;
+
     @Autowired
     private OCRService ocrService;
 
@@ -45,21 +74,15 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
     @BeforeEach
     @SneakyThrows
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void assertOCRModuleIsLoaded() {
 
         assert OCRModule.isModuleAvailable();
     }
 
 
-    @BeforeEach
-    public void stubMethods() {
-
-        when(dossierService.getDossier(TEST_DOSSIER_ID)).thenReturn(Dossier.builder().dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID).build());
-        when(dossierTemplateService.getDossierTemplate(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(DossierTemplate.builder().removeWatermark(false).build());
-    }
-
-
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testOCRMetrics() {
 
         testOCR("Watermark");
@@ -75,6 +98,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testOcr() {
         // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
         String text = testOCR("StitchedImagesMultiPage");
@@ -82,6 +106,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testManyRotatedImages() {
         // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
         String text = testOCR("manyRotatedImages");
@@ -90,6 +115,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testMergeImages() {
         // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
         String text = testOCR("merge_images");
@@ -110,6 +136,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testOCRWatermark() {
 
         assertThat(testOCR("Watermark")).contains("syngenta");
@@ -117,6 +144,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testOCRInvisibleText() {
 
         String text = testOCR("InvisibleText");
@@ -129,18 +157,66 @@ public class OcrServiceIntegrationTest extends AbstractTest {
     private String testOCR(String fileName) {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/" + fileName + ".pdf");
-        var originId = FileStorageService.getStorageId(TEST_DOSSIER_ID, "file", FileType.ORIGIN);
+        var originId = FileStorageService.getStorageId("dossier", "file", FileType.ORIGIN);
         try (var fileStream = pdfFileResource.getInputStream()) {
-            storageService.storeObject(TenantContext.getTenantId(), originId, fileStream);
+            storageService.storeObject(originId, fileStream);
         }
         try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
-            ocrService.runOcrOnDocument(TEST_DOSSIER_ID, "file", out);
-            System.out.println("File:" + getTemporaryDirectory() + "/" + fileName + ".pdf");
+            ocrService.runOcrOnDocument("dossier", "file", out);
         }
 
+        System.out.println("File:" + getTemporaryDirectory() + "/" + fileName + ".pdf");
         try (var fileStream = new FileInputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
             return extractAllTextFromDocument(fileStream);
         }
     }
 
+
+    private static String extractAllTextFromDocument(InputStream fileStream) throws IOException, PDFNetException {
+
+        PDFDoc pdfDoc = new PDFDoc(fileStream);
+        TextExtractor extractor = new TextExtractor();
+        List<String> texts = new ArrayList<>();
+
+        PageIterator iterator = pdfDoc.getPageIterator();
+        while (iterator.hasNext()) {
+            Page page = iterator.next();
+            extractor.begin(page);
+            texts.add(extractor.getAsText());
+        }
+
+        return String.join("\n", texts);
+    }
+
+
+    @SneakyThrows
+    public void dummyTest() {
+
+        // Build needs one test to not fail.
+        assertThat(1).isEqualTo(1);
+    }
+
+
+    @AfterEach
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
+    public static class TestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inMemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
 }

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/service/ImagePositionRetrievalServiceTest.java

@@ -1,8 +1,6 @@
 package com.iqser.red.service.ocr.v1.server.service;
 
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
-import static com.iqser.red.service.ocr.v1.server.utils.PdfDraw.drawGrid;
-import static com.iqser.red.service.ocr.v1.server.utils.PdfDraw.drawRectCollection;
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.io.FileInputStream;
@@ -15,24 +13,43 @@ import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.service.ocr.v1.server.AbstractTest;
+import com.iqser.red.service.ocr.v1.server.Application;
+import com.iqser.red.service.ocr.v1.server.OcrServiceIntegrationTest;
 import com.pdftron.common.PDFNetException;
+import com.pdftron.pdf.ColorPt;
+import com.pdftron.pdf.ColorSpace;
+import com.pdftron.pdf.Element;
+import com.pdftron.pdf.ElementBuilder;
 import com.pdftron.pdf.ElementWriter;
 import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.pdf.Rect;
 import com.pdftron.pdf.RectCollection;
 import com.pdftron.sdf.SDFDoc;
 
 import lombok.SneakyThrows;
 
-
-class ImagePositionRetrievalServiceTest extends AbstractTest {
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(OcrServiceIntegrationTest.TestConfiguration.class)
+class ImagePositionRetrievalServiceTest {
 
     @Autowired
     private ImagePositionRetrievalService imagePositionRetrievalService;
 
+    @MockBean
+    protected RabbitTemplate rabbitTemplate;
+
+
     @Test
     @SneakyThrows
     public void testImagePositionRetrievalForRotateTestFileWithImages() {
@@ -116,38 +133,37 @@ class ImagePositionRetrievalServiceTest extends AbstractTest {
 
     private List<int[]> testImagePositionDetection(String fileName) throws IOException, PDFNetException {
 
-        try (InputStream fileStream = new FileInputStream(new ClassPathResource("files/" + fileName + ".pdf").getFile().getAbsolutePath())) {
-            PDFDoc pdfDoc = new PDFDoc(fileStream);
+        InputStream fileStream = new FileInputStream(new ClassPathResource("files/" + fileName + ".pdf").getFile().getAbsolutePath());
+        PDFDoc pdfDoc = new PDFDoc(fileStream);
 
-            Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, false);
+        Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, false);
 
-            ElementWriter writer = new ElementWriter();
-            pageIdToRectCollection.forEach((pageId, rectCollection) -> {
-                try {
-                    writer.begin(pdfDoc.getPage(pageId));
-                    drawRectCollection(writer, rectCollection);
-                    drawGrid(writer, pdfDoc.getPage(pageId));
-                    writer.end();
-                    StringBuilder zonesString = new StringBuilder();
-                    for (int j = 0; j < rectCollection.getNumRects(); ++j) {
-                        var r = rectCollection.getRectAt(j);
-                        zonesString.append(String.format("%d:[lower left (%.1f|%.1f) upper right (%.1f|%.1f)]; ", j, r.getX1(), r.getY1(), r.getX2(), r.getY2()));
-                    }
-                    System.out.printf("Page %d: rotation: %d OCR regions %s \n", pageId, pdfDoc.getPage(pageId).getRotation(), zonesString);
-                } catch (PDFNetException e) {
-                    throw new RuntimeException(e);
+        ElementWriter writer = new ElementWriter();
+        pageIdToRectCollection.forEach((pageId, rectCollection) -> {
+            try {
+                writer.begin(pdfDoc.getPage(pageId));
+                drawRectCollection(writer, rectCollection, pdfDoc.getPage(pageId));
+                drawGrid(writer, pdfDoc.getPage(pageId));
+                writer.end();
+                StringBuilder zonesString = new StringBuilder();
+                for (int j = 0; j < rectCollection.getNumRects(); ++j) {
+                    var r = rectCollection.getRectAt(j);
+                    zonesString.append(String.format("%d:[lower left (%.1f|%.1f) upper right (%.1f|%.1f)]; ", j, r.getX1(), r.getY1(), r.getX2(), r.getY2()));
                 }
-            });
-
-            // Check visually for red Rectangles to match images in the saved pdf file
-            try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + "_IMAGE_BBOX.pdf")) {
-                out.write(pdfDoc.save(SDFDoc.SaveMode.LINEARIZED, null));
+                System.out.printf("Page %d: rotation: %d OCR regions %s \n", pageId, pdfDoc.getPage(pageId).getRotation(), zonesString);
+            } catch (PDFNetException e) {
+                throw new RuntimeException(e);
             }
-            pdfDoc.close();
-            System.out.println("result file: " + getTemporaryDirectory() + "/" + fileName + "_IMAGE_BBOX.pdf");
-            // round all coords to nearest int to account for inconsistencies with the calculation of the bounding box
-            return pageIdToRectCollection.values().stream().map(this::toRoundedCoordinateArrayList).flatMap(List::stream).collect(Collectors.toList());
+        });
+
+        // Check visually for red Rectangles to match images in the saved pdf file
+        try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + "_IMAGE_BBOX.pdf")) {
+            out.write(pdfDoc.save(SDFDoc.SaveMode.LINEARIZED, null));
         }
+        System.out.println("result file: " + getTemporaryDirectory() + "/" + fileName + "_IMAGE_BBOX.pdf");
+        fileStream.close();
+        // round all coords to nearest int to account for inconsistencies with the calculation of the bounding box
+        return pageIdToRectCollection.values().stream().map(this::toRoundedCoordinateArrayList).flatMap(List::stream).collect(Collectors.toList());
     }
 
 
@@ -162,4 +178,59 @@ class ImagePositionRetrievalServiceTest extends AbstractTest {
         return coords;
     }
 
+
+    @SneakyThrows
+    private void drawGrid(ElementWriter writer, Page page) {
+
+        ElementBuilder eb = new ElementBuilder();
+        double dX = 15;
+        double dY = 15;
+        int nRows = (int) (page.getPageHeight() / dY) + 1;
+        int nCols = (int) (page.getPageWidth() / dX) + 1;
+        for (int row = 0; row < nRows; ++row) {
+            for (int col = 0; col < nCols; ++col) {
+                Element cell = eb.createRect(col * dX, row * dY, dX, dY);
+                cell.setPathStroke(true);
+                cell.getGState().setLineWidth(1);
+                cell.getGState().setStrokeOpacity(0.1);
+                cell.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+                if (row == 0 && col == 0) {
+                    cell.getGState().setStrokeColor(new ColorPt(0, 0, 1));
+                    cell.setPathFill(true);
+                    cell.getGState().setFillOpacity(0.8);
+                    cell.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
+                    cell.getGState().setFillColor(new ColorPt(0, 0, 1));
+                } else {
+                    cell.setPathFill(false);
+                    cell.getGState().setStrokeColor(new ColorPt(0.1, 0.1, 0.1));
+                }
+                writer.writePlacedElement(cell);
+            }
+        }
+    }
+
+
+    @SneakyThrows
+    public static void drawRectCollection(ElementWriter writer, RectCollection rectCollection, Page page) {
+
+        ColorPt colorPt = new ColorPt(1, 0, 0);
+        ElementBuilder eb = new ElementBuilder();
+        for (int i = 0; i < rectCollection.getNumRects(); ++i) {
+            Rect r = rectCollection.getRectAt(i);
+            Element rect = eb.createRect(r.getX1(), r.getY1(), r.getWidth(), r.getHeight());
+
+            rect.setPathStroke(true);
+            rect.getGState().setLineWidth(5);
+            rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
+            rect.getGState().setStrokeColor(colorPt);
+
+            rect.setPathFill(true);
+            rect.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
+            rect.getGState().setFillColor(colorPt);
+            rect.getGState().setFillOpacity(0.5);
+
+            writer.writePlacedElement(rect);
+        }
+    }
+
 }

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/service/InvisibleElementRemovalServiceTest.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.ocr.v1.server.service;
 
-import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 
@@ -8,19 +7,34 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Bean;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
-import com.iqser.red.service.ocr.v1.server.AbstractTest;
+import com.iqser.red.service.ocr.v1.server.Application;
+import com.iqser.red.service.ocr.v1.server.OcrServiceIntegrationTest;
+import com.pdftron.pdf.PDFDoc;
+import com.pdftron.pdf.Page;
+import com.pdftron.pdf.PageIterator;
+import com.pdftron.pdf.TextExtractor;
 
 import lombok.SneakyThrows;
 
-public class InvisibleElementRemovalServiceTest extends AbstractTest {
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(OcrServiceIntegrationTest.TestConfiguration.class)
+public class InvisibleElementRemovalServiceTest {
 
-   @Autowired
-   private InvisibleElementRemovalService invisibleElementRemovalService;
+    @Autowired
+    private InvisibleElementRemovalService invisibleElementRemovalService;
+
+    @MockBean
+    protected RabbitTemplate rabbitTemplate;
 
 
     @Test
@@ -32,20 +46,28 @@ public class InvisibleElementRemovalServiceTest extends AbstractTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/" + fileName + ".pdf");
 
         try (var initialFileStream = pdfFileResource.getInputStream(); var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
-            invisibleElementRemovalService.removeInvisibleElements(initialFileStream, out, false);
+            invisibleElementRemovalService.removeInvisibleElements(initialFileStream, out,false);
         }
 
         try (var initialFileStream = pdfFileResource.getInputStream(); var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + "_delta.pdf")) {
-            invisibleElementRemovalService.removeInvisibleElements(initialFileStream, out, true);
+            invisibleElementRemovalService.removeInvisibleElements(initialFileStream, out,true);
         }
 
         System.out.println("Output File without invisible elements: files/" + fileName + ".pdf");
         System.out.println("Output Delta File without invisible elements: files/" + fileName + "_delta.pdf");
+        TextExtractor extractor = new TextExtractor();
 
-        try (var fileStream = new FileInputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
-            String[] text = extractAllTextFromDocument(fileStream).split("\n");
-            assertThat(text).contains("APPENDIX 16 Pathology Report", "Amendment 1", "Page 255 of 260");
+        PDFDoc pdfDoc;
+        try(var fileStream = new FileInputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
+            pdfDoc = new PDFDoc(fileStream);
         }
 
+        PageIterator iterator = pdfDoc.getPageIterator();
+        while (iterator.hasNext()) {
+            Page page = iterator.next();
+            extractor.begin(page);
+            String[] text = extractor.getAsText().split("\n");
+            assertThat(text).contains("APPENDIX 16 Pathology Report", "Amendment 1", "Page 255 of 260");
+        }
     }
 }

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/service/WatermarkRemovalServiceTest.java

@@ -1,31 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.service;
-
-import java.io.FileOutputStream;
-
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
-import com.iqser.red.pdftronlogic.commons.WatermarkRemovalService;
-import com.iqser.red.service.ocr.v1.server.AbstractTest;
-import com.iqser.red.service.ocr.v1.server.utils.OsUtils;
-
-import lombok.SneakyThrows;
-
-class WatermarkRemovalServiceTest extends AbstractTest {
-
-    @Autowired
-    private WatermarkRemovalService watermarkRemovalService;
-
-
-    @Test
-    @SneakyThrows
-    public void removeWatermarksTest() {
-
-        String filename = "files/Watermark.pdf";
-        try (var in = new ClassPathResource(filename).getInputStream(); var out = new FileOutputStream(OsUtils.createTmpFileName(filename, "WATERMARK_REMOVED"))) {
-            watermarkRemovalService.removeWatermarks(in, out);
-        }
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/utils/FileSystemBackedStorageService.java

@@ -32,7 +32,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @SneakyThrows
     @Override
-    public InputStreamResource getObject(String tenantId, String objectId) {
+    public InputStreamResource getObject(String objectId) {
 
         var res = dataMap.get(objectId);
         if (res == null) {
@@ -44,22 +44,28 @@ public class FileSystemBackedStorageService implements StorageService {
 
 
     @Override
-    public void deleteObject(String tenantId, String objectId) {
+    public void deleteObject(String objectId) {
 
         dataMap.remove(objectId);
     }
 
 
     @Override
-    public boolean objectExists(String tenantId, String objectId) {
+    public boolean objectExists(String objectId) {
 
         return dataMap.containsKey(objectId);
     }
 
 
+    @Override
+    public void init() {
+
+    }
+
+
     @Override
     @SneakyThrows
-    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
+    public <T> void storeJSONObject(String objectId, T any) {
 
         File tempFile = File.createTempFile("test", ".tmp");
         getMapper().writeValue(new FileOutputStream(tempFile), any);
@@ -75,7 +81,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @Override
     @SneakyThrows
-    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
+    public <T> T readJSONObject(String objectId, Class<T> clazz) {
 
         if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
             throw new StorageObjectDoesNotExist("Stored object not found");
@@ -98,7 +104,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @Override
     @SneakyThrows
-    public void storeObject(String tenantId, String objectId, InputStream stream) {
+    public void storeObject(String objectId, InputStream stream) {
 
         File tempFile = File.createTempFile("test", ".tmp");
 

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/utils/OsUtils.java

@@ -1,8 +1,5 @@
 package com.iqser.red.service.ocr.v1.server.utils;
 
-import java.nio.file.Path;
-import java.util.stream.Stream;
-
 import org.apache.commons.lang3.StringUtils;
 
 import lombok.SneakyThrows;
@@ -58,8 +55,4 @@ public final class OsUtils {
         return "/tmp";
     }
 
-    public static String createTmpFileName(String filename, String suffix) {
-
-        return Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(filename).getFileName()).toString().replace(".pdf", "_" + suffix + ".pdf");
-    }
 }

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/utils/PdfDraw.java

@@ -1,74 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.utils;
-
-import com.pdftron.pdf.ColorPt;
-import com.pdftron.pdf.ColorSpace;
-import com.pdftron.pdf.Element;
-import com.pdftron.pdf.ElementBuilder;
-import com.pdftron.pdf.ElementWriter;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.Rect;
-import com.pdftron.pdf.RectCollection;
-
-import lombok.SneakyThrows;
-
-
-public class PdfDraw {
-
-    @SneakyThrows
-    public static void drawGrid(ElementWriter writer, Page page) {
-
-        ElementBuilder eb = new ElementBuilder();
-        double dX = 15;
-        double dY = 15;
-        int nRows = (int) (page.getPageHeight() / dY) + 1;
-        int nCols = (int) (page.getPageWidth() / dX) + 1;
-        for (int row = 0; row < nRows; ++row) {
-            for (int col = 0; col < nCols; ++col) {
-                Element cell = eb.createRect(col * dX, row * dY, dX, dY);
-                cell.setPathStroke(true);
-                cell.getGState().setLineWidth(1);
-                cell.getGState().setStrokeOpacity(0.1);
-                cell.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
-                if (row == 0 && col == 0) {
-                    cell.getGState().setStrokeColor(new ColorPt(0, 0, 1));
-                    cell.setPathFill(true);
-                    cell.getGState().setFillOpacity(0.8);
-                    cell.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
-                    cell.getGState().setFillColor(new ColorPt(0, 0, 1));
-                } else {
-                    cell.setPathFill(false);
-                    cell.getGState().setStrokeColor(new ColorPt(0.1, 0.1, 0.1));
-                }
-                writer.writePlacedElement(cell);
-            }
-        }
-        eb.destroy();
-    }
-
-
-    @SneakyThrows
-    public static void drawRectCollection(ElementWriter writer, RectCollection rectCollection) {
-
-        ColorPt colorPt = new ColorPt(1, 0, 0);
-        ElementBuilder eb = new ElementBuilder();
-        for (int i = 0; i < rectCollection.getNumRects(); ++i) {
-            Rect r = rectCollection.getRectAt(i);
-            Element rect = eb.createRect(r.getX1(), r.getY1(), r.getWidth(), r.getHeight());
-
-            rect.setPathStroke(true);
-            rect.getGState().setLineWidth(5);
-            rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
-            rect.getGState().setStrokeColor(colorPt);
-
-            rect.setPathFill(true);
-            rect.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
-            rect.getGState().setFillColor(colorPt);
-            rect.getGState().setFillOpacity(0.5);
-
-            writer.writePlacedElement(rect);
-        }
-        colorPt.destroy();
-        eb.destroy();
-    }
-
-}

ocr-service-v1/pom.xml

@@ -7,13 +7,13 @@
     <parent>
         <groupId>com.iqser.red</groupId>
         <artifactId>platform-dependency</artifactId>
-        <version>2.2.0</version>
+        <version>1.16.0</version>
         <relativePath/>
     </parent>
 
     <groupId>com.iqser.red.service</groupId>
     <artifactId>ocr-service-v1</artifactId>
-    <version>3.0-SNAPSHOT</version>
+    <version>1.0-SNAPSHOT</version>
 
     <packaging>pom</packaging>
 
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>2.6.0</version>
+                <version>1.20.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>
@@ -39,12 +39,12 @@
                 <plugin>
                     <groupId>org.sonarsource.scanner.maven</groupId>
                     <artifactId>sonar-maven-plugin</artifactId>
-                    <version>3.9.1.2184</version>
+                    <version>3.9.0.2155</version>
                 </plugin>
                 <plugin>
                     <groupId>org.owasp</groupId>
                     <artifactId>dependency-check-maven</artifactId>
-                    <version>8.1.2</version>
+                    <version>6.3.1</version>
                     <configuration>
                         <format>ALL</format>
                     </configuration>

pom.xml

@@ -6,11 +6,12 @@
 
     <groupId>com.iqser.red.service</groupId>
     <artifactId>ocr-service</artifactId>
-    <version>3.0-SNAPSHOT</version>
+    <version>1.0-SNAPSHOT</version>
 
     <packaging>pom</packaging>
 
     <modules>
+        <module>bamboo-specs</module>
         <module>ocr-service-v1</module>
         <module>ocr-service-image-v1</module>
     </modules>