Merge branch 'RED-6072-timestamp' into 'master'

RED-6072 - As Operation I want to see why files are in an ERROR state Closes RED-6072 See merge request redactmanager/ocr-service!2
RED-6072 - As Operation I want to see why files are in an ERROR state
2023-06-15 14:08:39 +02:00 · 2023-06-15 14:37:32 +03:00 · 2023-05-31 15:44:02 +02:00 · 2023-05-31 15:10:37 +02:00 · 2023-05-25 14:59:04 +02:00 · 2023-05-25 14:19:18 +02:00
26 changed files with 106 additions and 1101 deletions
@@ -0,0 +1,6 @@
+variables:
+  SONAR_PROJECT_KEY: 'LM_license-service'
+include:
+  - project: 'gitlab/gitlab'
+    ref: 'main'
+    file: 'ci-templates/maven_java.yml'
@@ -1,37 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.atlassian.bamboo</groupId>
-        <artifactId>bamboo-specs-parent</artifactId>
-        <version>8.1.3</version>
-        <relativePath/>
-    </parent>
-
-    <artifactId>bamboo-specs</artifactId>
-    <version>1.0.0-SNAPSHOT</version>
-    <packaging>jar</packaging>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.atlassian.bamboo</groupId>
-            <artifactId>bamboo-specs-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.atlassian.bamboo</groupId>
-            <artifactId>bamboo-specs</artifactId>
-        </dependency>
-
-        <!-- Test dependencies -->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-
-    <!-- run 'mvn test' to perform offline validation of the plan -->
-    <!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
-</project>
@@ -1,125 +0,0 @@
-package buildjob;
-
-import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
-
-import java.time.LocalTime;
-
-import com.atlassian.bamboo.specs.api.BambooSpec;
-import com.atlassian.bamboo.specs.api.builders.BambooKey;
-import com.atlassian.bamboo.specs.api.builders.Variable;
-import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
-import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
-import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
-import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions;
-import com.atlassian.bamboo.specs.api.builders.plan.Job;
-import com.atlassian.bamboo.specs.api.builders.plan.Plan;
-import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
-import com.atlassian.bamboo.specs.api.builders.plan.Stage;
-import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
-import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
-import com.atlassian.bamboo.specs.api.builders.project.Project;
-import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
-import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
-import com.atlassian.bamboo.specs.builders.task.ScriptTask;
-import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
-import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
-import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
-import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
-import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
-import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
-import com.atlassian.bamboo.specs.util.BambooServer;
-
-/**
- * Plan configuration for Bamboo.
- * Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
- */
-@BambooSpec
-public class PlanSpec {
-
-    private static final String SERVICE_NAME = "ocr-service";
-
-    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
-
-
-    /**
-     * Run main to publish plan on Bamboo
-     */
-    public static void main(final String[] args) throws Exception {
-        //By default credentials are read from the '.credentials' file.
-        BambooServer bambooServer = new BambooServer("http://localhost:8085");
-
-        Plan plan = new PlanSpec().createPlan();
-        bambooServer.publish(plan);
-        PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
-        bambooServer.publish(planPermission);
-
-        Plan secPlan = new PlanSpec().createSecBuild();
-        bambooServer.publish(secPlan);
-        PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
-        bambooServer.publish(secPlanPermission);
-    }
-
-
-    private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
-
-        Permissions permission = new Permissions().userPermissions("atlbamboo",
-                        PermissionType.EDIT,
-                        PermissionType.VIEW,
-                        PermissionType.ADMIN,
-                        PermissionType.CLONE,
-                        PermissionType.BUILD)
-                .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
-                .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
-                .loggedInUserPermissions(PermissionType.VIEW)
-                .anonymousUserPermissionView();
-        return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
-    }
-
-
-    private Project project() {
-
-        return new Project().name("RED").key(new BambooKey("RED"));
-    }
-
-
-    public Plan createPlan() {
-
-        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
-                .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                                        .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
-                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
-                                new ScriptTask().description("Build").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh").argument(SERVICE_NAME),
-                                createJUnitParserTask().description("Resultparser")
-                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                        .enabled(true),
-                                new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL),
-                                new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(new BitbucketServerTrigger())
-                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
-                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
-                        .notificationForCommitters());
-    }
-
-
-    public Plan createSecBuild() {
-
-        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                                        .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
-                                new VcsCheckoutTask().description("Checkout Default Repository").checkoutItems(new CheckoutItem().defaultRepository()),
-                                new ScriptTask().description("Sonar").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh").argument(SERVICE_NAME))
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
-                                .dockerRunArguments("--net=host")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
-                .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
-    }
-
-}
@@ -1,62 +0,0 @@
-#!/bin/bash
-set -e
-
-SERVICE_NAME=$1
-
-if [[ "$bamboo_planRepository_branchName" == "master" ]]
-then
-	echo "building on master branch"
-    branchVersion=$(cat pom.xml | grep -Eo "<version>.*" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|')
-	echo "branch version is : $branchVersion"
-    latestVersion=$(semver $( git tag -l $branchVersion.* ) | tail -n1)
-	echo "latestVersion is : $latestVersion"
-    newVersion="$(semver $latestVersion -p -i minor)"
-	echo "newVersion is : $newVersion"
-elif [[ "$bamboo_planRepository_branchName" == release* ]]
-then
-    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
-    latestVersion=$(semver $( git tag -l $branchVersion.* ) | tail -n1)
-    newVersion="$(semver $latestVersion -p -i patch)"
-elif [[ "${bamboo_version_tag}" != "dev" ]]
-then
-    newVersion="${bamboo_version_tag}"
-else
-    mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        --no-transfer-progress \
-        ${bamboo_maven_add_param} \
-        clean install \
-        -Djava.security.egd=file:/dev/./urandomelse
-    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
-    exit 0
-fi
-
-echo "gitTag=${newVersion}" > git.tag
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    versions:set \
-    -DnewVersion=${newVersion}
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    versions:set  \
-    -DnewVersion=${newVersion}
-
-mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    --no-transfer-progress \
-    clean deploy \
-    ${bamboo_maven_add_param} \
-    -e \
-    -DdeployAtEnd=true \
-    -Dmaven.wagon.http.ssl.insecure=true \
-    -Dmaven.wagon.http.ssl.allowall=true \
-    -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    package
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    docker:push
@@ -1,44 +0,0 @@
-#!/bin/bash
-set -e
-
-SERVICE_NAME=$1
-
-echo "build jar binaries"
-mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    --no-transfer-progress \
-    clean install \
-    -Djava.security.egd=file:/dev/./urandomelse
-
-echo "dependency-check:aggregate"
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    org.owasp:dependency-check-maven:aggregate
-
-if [[ -z "${bamboo_repository_pr_key}" ]]
-then
-    echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
-    mvn --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        sonar:sonar \
-        -Dsonar.projectKey=RED_$SERVICE_NAME \
-        -Dsonar.host.url=https://sonarqube.iqser.com \
-        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-        -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
-        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-else
-    echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
-    mvn --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        sonar:sonar \
-        -Dsonar.projectKey=RED_$SERVICE_NAME \
-        -Dsonar.host.url=https://sonarqube.iqser.com \
-        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-        -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
-        -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
-        -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
-        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-fi
@@ -1,22 +0,0 @@
-package buildjob;
-
-import org.junit.Test;
-
-import com.atlassian.bamboo.specs.api.builders.plan.Plan;
-import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException;
-import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
-
-public class PlanSpecTest {
-
-    @Test
-    public void checkYourPlanOffline() throws PropertiesValidationException {
-
-        Plan plan = new PlanSpec().createPlan();
-        EntityPropertiesBuilders.build(plan);
-
-        Plan secPlan = new PlanSpec().createSecBuild();
-        EntityPropertiesBuilders.build(secPlan);
-
-    }
-
-}
@@ -3,16 +3,16 @@
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
-        <groupId>com.iqser.red</groupId>
+        <groupId>com.knecon.fforesight</groupId>
        <artifactId>platform-docker-dependency</artifactId>
-        <version>1.2.0</version>
+        <version>0.1.0</version>
        <relativePath/>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>ocr-service-image-v1</artifactId>
    <groupId>com.iqser.red.service</groupId>
-    <version>1.0-SNAPSHOT</version>
+    <version>3.0-SNAPSHOT</version>
    <packaging>pom</packaging>

    <properties>
@@ -7,26 +7,17 @@
    <parent>
        <groupId>com.iqser.red.service</groupId>
        <artifactId>ocr-service-v1</artifactId>
-        <version>1.0-SNAPSHOT</version>
+        <version>3.0-SNAPSHOT</version>
    </parent>

    <artifactId>ocr-service-api-v1</artifactId>
-    <version>1.0-SNAPSHOT</version>

    <properties>
-        <persistence-service.version>2.1.0</persistence-service.version>
-        <dsljson.version>1.9.9</dsljson.version>
+        <persistence-service.version>2.70.0</persistence-service.version>
    </properties>

    <dependencies>

-        <!-- https://mvnrepository.com/artifact/com.dslplatform/dsl-json-java8 -->
-        <dependency>
-            <groupId>com.dslplatform</groupId>
-            <artifactId>dsl-json-java8</artifactId>
-            <version>${dsljson.version}</version>
-        </dependency>
-
        <dependency>
            <!-- This dependency contains annotations that are used in specifying REST endpoints. -->
            <!-- It is optional since not all users of this API might use Feign. -->
@@ -7,11 +7,10 @@
    <parent>
        <groupId>com.iqser.red.service</groupId>
        <artifactId>ocr-service-v1</artifactId>
-        <version>1.0-SNAPSHOT</version>
+        <version>3.0-SNAPSHOT</version>
    </parent>

    <artifactId>ocr-service-server-v1</artifactId>
-    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
@@ -24,6 +23,12 @@
            <groupId>com.iqser.red.commons</groupId>
            <artifactId>storage-commons</artifactId>
        </dependency>
+        <dependency>
+            <groupId>com.iqser.red.commons</groupId>
+            <artifactId>pdftron-logic-commons</artifactId>
+            <version>2.1.0</version>
+        </dependency>
+
        <dependency>
            <groupId>com.iqser.red.commons</groupId>
            <artifactId>spring-commons</artifactId>
@@ -39,17 +44,16 @@
        <dependency>
            <groupId>com.pdftron</groupId>
            <artifactId>PDFNet</artifactId>
-            <version>9.4.0</version>
+            <version>10.1.0</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-amqp</artifactId>
-            <version>2.3.1.RELEASE</version>
        </dependency>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-kms</artifactId>
-            <version>1.12.158</version>
+            <version>1.12.440</version>
        </dependency>
        <dependency>
            <groupId>com.google.guava</groupId>
@@ -65,7 +69,6 @@
        <dependency>
            <groupId>org.springframework.amqp</groupId>
            <artifactId>spring-rabbit-test</artifactId>
-            <version>2.3.1</version>
            <scope>test</scope>
        </dependency>
        <dependency>
@@ -81,7 +84,6 @@
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
-            <version>3.12.0</version>
        </dependency>
    </dependencies>
    <build>
@@ -110,7 +112,6 @@
                <configuration>
                    <annotationProcessors>
                        <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
-                        <annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
                    </annotationProcessors>
                </configuration>
            </plugin>
@@ -10,12 +10,14 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.annotation.EnableAsync;

+import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
 import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.ocr.v1.server.multitenancy.AsyncConfig;
 import com.iqser.red.service.ocr.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
 import com.iqser.red.service.ocr.v1.server.multitenancy.MultiTenancyWebConfiguration;
 import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;

 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
@@ -23,7 +25,7 @@ import io.micrometer.core.instrument.MeterRegistry;
@EnableAsync
@EnableConfigurationProperties(OcrServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
-@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class})
+@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class, StorageAutoConfiguration.class})
@EnableFeignClients(basePackageClasses = FileStatusProcessingUpdateClient.class)
 public class Application {

@@ -44,4 +46,11 @@ public class Application {
        return new TimedAspect(registry);
    }

+
+    @Bean
+    public InvisibleElementRemovalService invisibleElementRemovalService() {
+
+        return new InvisibleElementRemovalService();
+    }
+
 }
@@ -20,6 +20,9 @@ public class MessagingConfiguration {

    public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue";

+    public static final String X_ERROR_INFO_HEADER = "x-error-message";
+    public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
+

    @Bean
    public Queue ocrQueue() {
@@ -1,13 +1,13 @@
 package com.iqser.red.service.ocr.v1.server.initializer;

 import com.pdftron.pdf.PDFNet;
+
+import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;

-import javax.annotation.PostConstruct;
-
@Component
@RequiredArgsConstructor
 public class PDFNetInitializer {
@@ -1,68 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.model;
-
-import static com.iqser.red.service.ocr.v1.server.service.InvisibleElementRemovalService.TOLERANCE;
-
-import java.awt.geom.Area;
-import java.awt.geom.GeneralPath;
-import java.awt.geom.Rectangle2D;
-import java.util.Deque;
-import java.util.LinkedList;
-
-import com.pdftron.pdf.Rect;
-
-import lombok.Data;
-import lombok.SneakyThrows;
-
-@Data
-public class ClippingPathStack {
-
-    private Deque<Area> stack = new LinkedList<>();
-
-
-    @SneakyThrows
-    public ClippingPathStack(Rect rectangle) {
-
-        stack.push(new Area(new Rectangle2D.Double(rectangle.getX1(), rectangle.getY1(), rectangle.getWidth(), rectangle.getHeight()).getBounds2D()));
-    }
-
-
-    @SneakyThrows
-    public void intersectClippingPath(GeneralPath path) {
-
-        getCurrentClippingPath().intersect(new Area(path));
-    }
-
-
-    public boolean almostIntersects(double x, double y, double width, double height) {
-        // To address inconsistencies in the calculation of the bounding box we slightly increase the rectangle
-        // Height or width are zero for straight lines, even though they are being rendered. Therefore, height or width must be at minimum >0.
-
-        double x_with_tolerance = x > 0 ? x - TOLERANCE : x + TOLERANCE;
-        double y_with_tolerance = y > 0 ? y - TOLERANCE : y + TOLERANCE;
-        double width_with_tolerance = width + (2 * TOLERANCE);
-        double height_with_tolerance = height + (2 * TOLERANCE);
-        return getCurrentClippingPath().intersects(x_with_tolerance, y_with_tolerance, width_with_tolerance, height_with_tolerance);
-    }
-
-
-    public Area getCurrentClippingPath() {
-
-        return stack.peek();
-    }
-
-
-    public void enterNewGState() {
-
-        Area current = stack.peek();
-        Area cloned = new Area();
-        cloned.add(current);
-        stack.push(cloned);
-    }
-
-
-    public void leaveGState() {
-
-        stack.pop();
-    }
-
-}
@@ -1,170 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.model;
-
-import static com.iqser.red.service.ocr.v1.server.service.InvisibleElementRemovalService.TOLERANCE;
-
-import java.awt.geom.Rectangle2D;
-
-import com.pdftron.common.PDFNetException;
-import com.pdftron.pdf.Element;
-import com.pdftron.pdf.Rect;
-
-import lombok.AccessLevel;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
-import lombok.experimental.SuperBuilder;
-
-@Getter
-@SuperBuilder
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ElementFeatures {
-
-    int elementType;
-    Rectangle2D boundingBox;
-
-
-    public boolean almostMatches(Element element) throws PDFNetException {
-
-        return element.getType() == elementType && //
-                element.getBBox() != null && //
-                rectsAlmostMatch(element.getBBox());
-    }
-
-
-    protected boolean almostEqual(double a, double b) {
-
-        return Math.abs(a - b) < TOLERANCE;
-    }
-
-
-    @SneakyThrows
-    private boolean rectsAlmostMatch(Rect bBox) {
-        // To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance
-
-        return almostEqual(bBox.getX1(), boundingBox.getX()) && //
-                almostEqual(bBox.getY1(), boundingBox.getY()) && //
-                almostEqual(bBox.getWidth(), boundingBox.getWidth()) && //
-                almostEqual(bBox.getHeight(), boundingBox.getHeight());
-    }
-
-
-    @EqualsAndHashCode(callSuper = true)
-    @Getter
-    @SuperBuilder
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    private static class Text extends ElementFeatures {
-
-        String text;
-        int font;
-        double fontsize;
-
-
-        @Override
-        public boolean almostMatches(Element element) throws PDFNetException {
-
-            return super.almostMatches(element) && //
-                    text.equals(element.getTextString()) && //
-                    font == element.getGState().getFont().getType() && //
-                    almostEqual(fontsize, element.getGState().getFontSize());
-        }
-
-    }
-
-    @EqualsAndHashCode(callSuper = true)
-    @Getter
-    @SuperBuilder
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    private static class Path extends ElementFeatures {
-
-        boolean isClippingPath;
-        boolean isClipWindingFill;
-        boolean isStroked;
-        boolean isFilled;
-        boolean isWindingFill;
-
-
-        @Override
-        public boolean almostMatches(Element element) throws PDFNetException {
-
-            return super.almostMatches(element) && //
-                    isClippingPath == element.isClippingPath() && //
-                    isClipWindingFill == element.isClipWindingFill() && //
-                    isStroked == element.isStroked() && //
-                    isFilled == element.isFilled() && //
-                    isWindingFill == element.isWindingFill();
-
-        }
-
-    }
-
-    @EqualsAndHashCode(callSuper = true)
-    @Getter
-    @SuperBuilder
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    private static class Image extends ElementFeatures {
-
-        int dataSize;
-        int height;
-        int width;
-        int renderingIntent;
-        int componentNum;
-        int bitsPerComponent;
-
-
-        @Override
-        public boolean almostMatches(Element element) throws PDFNetException {
-
-            return super.almostMatches(element) && //
-                    dataSize == element.getImageDataSize() && //
-                    height == element.getImageHeight() && //
-                    width == element.getImageWidth() && //
-                    renderingIntent == element.getImageRenderingIntent() && //
-                    componentNum == element.getComponentNum() && //
-                    bitsPerComponent == element.getBitsPerComponent();
-        }
-
-    }
-
-
-    public static ElementFeatures extractFeatures(Element element) throws PDFNetException {
-
-        return switch (element.getType()) {
-            case Element.e_path -> Path.builder()
-                    .elementType(element.getType())
-                    .boundingBox(toRectangle2D(element.getBBox()))
-                    .isClippingPath(element.isClippingPath())
-                    .isClipWindingFill(element.isClipWindingFill())
-                    .isStroked(element.isStroked())
-                    .isFilled(element.isFilled())
-                    .isWindingFill(element.isWindingFill())
-                    .build();
-            case Element.e_text -> Text.builder()
-                    .elementType(element.getType())
-                    .boundingBox(toRectangle2D(element.getBBox()))
-                    .text(element.getTextString())
-                    .font(element.getGState().getFont().getType())
-                    .fontsize(element.getGState().getFontSize())
-                    .build();
-            case Element.e_image, Element.e_inline_image -> Image.builder()
-                    .elementType(element.getType())
-                    .boundingBox(toRectangle2D(element.getBBox()))
-                    .dataSize(element.getImageDataSize())
-                    .height(element.getImageHeight())
-                    .width(element.getImageWidth())
-                    .renderingIntent(element.getImageRenderingIntent())
-                    .componentNum(element.getComponentNum())
-                    .bitsPerComponent(element.getBitsPerComponent())
-                    .build();
-            // This technically should never happen, it's a safetynet
-            default -> throw new RuntimeException("Feature Extraction is not supported for PDFTron.Element with type: " + element.getType());
-        };
-    }
-
-
-    private static Rectangle2D toRectangle2D(Rect rect) throws PDFNetException {
-
-        return new Rectangle2D.Double(rect.getX1(), rect.getY1(), rect.getWidth(), rect.getHeight());
-    }
-
-}
@@ -3,15 +3,12 @@ package com.iqser.red.service.ocr.v1.server.model.image;
 import java.util.ArrayList;
 import java.util.List;

-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;

 import lombok.Data;

@Data
-@CompiledJson
 public class ImageServiceResponse {

    private String dossierId;
@@ -19,13 +16,11 @@ public class ImageServiceResponse {

    @JsonProperty(value = "imageMetadata")
    @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
    private List<ImageMetadata> data = new ArrayList<>();


    @JsonProperty(value = "imageMetadata")
    @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
    public void setData(List<ImageMetadata> data) {this.data = data;}

 }
@@ -6,7 +6,6 @@ import java.security.SecureRandom;
 import java.security.spec.KeySpec;
 import java.util.Base64;

-import javax.annotation.PostConstruct;
 import javax.crypto.Cipher;
 import javax.crypto.SecretKey;
 import javax.crypto.SecretKeyFactory;
@@ -17,6 +16,7 @@ import javax.crypto.spec.SecretKeySpec;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;

+import jakarta.annotation.PostConstruct;
 import lombok.SneakyThrows;

@Service
@@ -1,466 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.service;
-
-import java.awt.Shape;
-import java.awt.geom.AffineTransform;
-import java.awt.geom.GeneralPath;
-import java.awt.geom.Rectangle2D;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.springframework.stereotype.Service;
-
-import com.google.common.primitives.Bytes;
-import com.google.common.primitives.Doubles;
-import com.iqser.red.service.ocr.v1.server.model.ClippingPathStack;
-import com.iqser.red.service.ocr.v1.server.model.ElementFeatures;
-import com.pdftron.common.Matrix2D;
-import com.pdftron.common.PDFNetException;
-import com.pdftron.pdf.ColorPt;
-import com.pdftron.pdf.ColorSpace;
-import com.pdftron.pdf.Element;
-import com.pdftron.pdf.ElementBuilder;
-import com.pdftron.pdf.ElementReader;
-import com.pdftron.pdf.ElementWriter;
-import com.pdftron.pdf.GState;
-import com.pdftron.pdf.PDFDoc;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.PageIterator;
-import com.pdftron.pdf.PathData;
-import com.pdftron.pdf.Rect;
-import com.pdftron.sdf.Obj;
-import com.pdftron.sdf.SDFDoc;
-
-import lombok.Builder;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-public class InvisibleElementRemovalService {
-
-    static public final double TOLERANCE = 1e-3;
-
-
-    /**
-     * Removes all hidden Text, Path and Image Elements from a PDF Document.
-     * handled cases:
-     * -Text which is transparent or is set to not render
-     * -Elements outside of clipping path
-     * -Elements that have been painted over by visible and filled Paths
-     * unhandled cases:
-     * -Elements covered by widely stroked path
-     * -Elements with the same color as background
-     * -Any Text set to clipping with its many interactions with other elements
-     *
-     * @param pdfFile The PDF file to process
-     * @param delta   If this flag is set only the removed Elements will be written to the output file.
-     *                The Elements are red if they are removed by clipping path, blue for transparency, and a green bounding box for overlap.
-     * @param out     OutputStream to write the resulting file to
-     **/
-    @SneakyThrows
-    public void removeInvisibleElements(InputStream pdfFile, OutputStream out, boolean delta) {
-
-        PDFDoc pdfDoc = new PDFDoc(pdfFile);
-
-        ElementWriter writer = new ElementWriter();
-        ElementReader reader = new ElementReader();
-        Set<Long> visitedXObjIds = new TreeSet<>();
-
-        for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
-
-            Page page = iterator.next();
-
-            visitedXObjIds.add(page.getSDFObj().getObjNum());
-
-
-            InvisibleElementRemovalContext context = InvisibleElementRemovalContext.builder()
-                    .reader(reader)
-                    .clippingPathStack(new ClippingPathStack(page.getMediaBox()))
-                    .delta(delta)
-                    .overlappedElements(new ArrayList<>())
-                    .visibleElements(new ArrayList<>())
-                    .visitedXObjIds(visitedXObjIds)
-                    .build();
-
-            removeClippedElementsAndInvisibleTextAndRememberOverlappedElements(page, writer, context);
-
-            context.visitedXObjIds().clear();
-
-            removeOverlappedElements(page, writer, context);
-        }
-
-        try {
-            pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
-        } catch (Exception e) {
-            log.error("File could not be saved after invisible element removal");
-            throw new RuntimeException(e);
-        }
-
-        writer.destroy();
-        reader.destroy();
-        pdfDoc.close();
-    }
-
-
-    private void removeClippedElementsAndInvisibleTextAndRememberOverlappedElements(Page page,
-                                                                                    ElementWriter writer,
-                                                                                    InvisibleElementRemovalContext context) throws PDFNetException {
-
-        context.reader().begin(page);
-        writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
-        processElements(writer, context);
-        writer.end();
-        context.reader().end();
-    }
-
-
-    private void processElements(ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
-
-        for (Element element = context.reader().next(); element != null; element = context.reader().next())
-            switch (element.getType()) {
-                case Element.e_image, Element.e_inline_image -> processImages(element, writer, context);
-                case Element.e_text -> processText(element, writer, context);
-                case Element.e_path -> processPath(element, writer, context);
-                case Element.e_form -> processForm(element, writer, context);
-                case Element.e_group_begin -> {
-                    context.clippingPathStack().enterNewGState();
-                    writer.writeElement(element);
-                }
-                case Element.e_group_end -> {
-                    context.clippingPathStack().leaveGState();
-                    writer.writeElement(element);
-                }
-                default -> writer.writeElement(element);
-            }
-    }
-
-
-    private void processImages(Element imageElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
-
-        Rect rect = imageElement.getBBox();
-
-        if (rect == null) {
-            return;
-        }
-
-        boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX1(), rect.getY1(), rect.getWidth(), rect.getHeight());
-
-        if (!context.delta() && inClippingPath) {
-            context.visibleElements().add(ElementFeatures.extractFeatures(imageElement));
-        }
-
-        if (context.delta() ^ inClippingPath) {
-            writer.writeElement(imageElement);
-        }
-    }
-
-
-    private void processText(Element textElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
-
-        Rect rect = textElement.getBBox();
-
-        if (rect == null) {
-            writer.writeElement(textElement);
-            return;
-        }
-
-        GState gState = textElement.getGState();
-
-        boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX1(), rect.getY1(), rect.getWidth(), rect.getHeight());
-
-        boolean isTextVisible = isTextRenderedVisibly(gState);
-
-        if (inClippingPath && isTextVisible) {
-            context.visibleElements().add(ElementFeatures.extractFeatures(textElement));
-        }
-        if (!context.delta()) {
-            if (inClippingPath && isTextVisible) {
-                writer.writeElement(textElement);
-            } else if (textElement.hasTextMatrix()) {
-                /*
-                PDFTron Element with type "text" refers to a Tj command. If a Tm command is just above it in the pdf file, PDFTron will join the two commands and treat them as one Element.
-                hasTextMatrix() checks for this case specifically. Also, Tm changes the position for a whole BT/ET segment, possibly containing multiple Tj commands.
-                Therefore, the position of a following Tj is affected by not writing the first Element.
-                This is why, we write only the Tm command:
-                */
-                writer.writeGStateChanges(textElement);
-            }
-        } else {
-            if (!inClippingPath) {
-                gState.setFillColorSpace(ColorSpace.createDeviceRGB());
-                // red for elements removed by clipping path
-                gState.setFillColor(new ColorPt(1, 0, 0));
-                writer.writeElement(textElement);
-            }
-            if (!isTextVisible) {
-                gState.setFillColorSpace(ColorSpace.createDeviceRGB());
-                // blue for elements removed due to transparency or not rendered
-                gState.setFillColor(new ColorPt(0, 0, 1));
-                gState.setTextRenderMode(GState.e_fill_text);
-                gState.setFillOpacity(1);
-                writer.writeElement(textElement);
-            }
-        }
-    }
-
-
-    private void processForm(Element formElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
-
-        writer.writeElement(formElement);
-        Obj formObj = formElement.getXObject();
-
-        if (!context.visitedXObjIds().contains(formObj.getObjNum())) {
-            context.visitedXObjIds().add(formObj.getObjNum());
-            // writer needs to be newly initialized when entering a new content stream
-            // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
-            ElementWriter formWriter = new ElementWriter();
-            context.reader().formBegin();
-            formWriter.begin(formObj);
-
-            context.reader().clearChangeList();
-            formWriter.setDefaultGState(context.reader());
-
-            processElements(formWriter, context);
-            formWriter.end();
-            formWriter.destroy();
-            context.reader().end();
-        }
-    }
-
-
-    private void processPath(Element pathElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
-
-        PathData pathData = pathElement.getPathData();
-
-        if (pathData.getOperators().length == 0 && pathData.getPoints().length == 0) {
-            writer.writeGStateChanges(pathElement);
-            return;
-        }
-
-        GeneralPath linePath = convertToGeneralPath(pathData);
-
-        //transform path to initial user space
-        var ctm = pathElement.getCTM();
-        var affineTransform = toAffineTransform(ctm);
-        linePath.transform(affineTransform);
-
-        var rect = linePath.getBounds2D();
-
-        boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX(), rect.getY(), rect.getWidth(), rect.getHeight());
-
-        if (pathElement.isClippingPath()) {
-            if (pathElement.isClipWindingFill()) {
-                linePath.setWindingRule(GeneralPath.WIND_NON_ZERO);
-            } else {
-                linePath.setWindingRule(GeneralPath.WIND_EVEN_ODD);
-            }
-
-            context.clippingPathStack().intersectClippingPath(linePath);
-            pathElement.setPathClip(!context.delta());
-            writer.writeElement(pathElement);
-
-        } else {
-            if (pathElement.isWindingFill()) {
-                linePath.setWindingRule(GeneralPath.WIND_NON_ZERO);
-            } else {
-                linePath.setWindingRule(GeneralPath.WIND_EVEN_ODD);
-            }
-
-            if (inClippingPath) {
-                if (isFilledAndNonTransparent(pathElement)) {
-                    List<ElementFeatures> currentOverlappedElements = context.visibleElements()
-                            .stream()
-                            .filter(features -> almostContains(linePath, features.getBoundingBox()))
-                            .toList();
-                    context.overlappedElements().addAll(currentOverlappedElements);
-                    context.visibleElements().removeAll(currentOverlappedElements);
-                }
-                context.visibleElements().add(ElementFeatures.extractFeatures(pathElement));
-                if (!context.delta()) {
-                    writer.writeElement(pathElement);
-                }
-            }
-            if (context.delta() && !inClippingPath) {
-                pathElement.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
-                pathElement.getGState().setFillColor(new ColorPt(1, 0, 0));
-                pathElement.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
-                pathElement.getGState().setStrokeColor(new ColorPt(1, 0, 0));
-                writer.writeElement(pathElement);
-            }
-        }
-    }
-
-
-    private void removeOverlappedElements(Page page, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
-
-        context.reader().begin(page);
-        writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
-        if (context.delta()) {
-            // green for element removed due to overlapping
-            context.overlappedElements().forEach(feature -> drawBBox(writer, feature.getBoundingBox(), "#00FF00"));
-            context.overlappedElements().clear();
-        }
-        processOverlappedElements(writer, context);
-        writer.end();
-        context.reader().end();
-
-        if (context.overlappedElements().size() > 0) {
-            log.warn(context.overlappedElements().size() + " overlapped elements have not been found or removed");
-        }
-    }
-
-
-    private void processOverlappedElements(ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
-
-        for (Element element = context.reader().next(); element != null; element = context.reader().next()) {
-            switch (element.getType()) {
-                case Element.e_form -> processFormOverlappedElements(writer, element, context);
-                case Element.e_path, Element.e_image, Element.e_inline_image, Element.e_text -> {
-                    boolean anyMatch = false;
-                    for (ElementFeatures elementToRemove : context.overlappedElements()) {
-                        if (elementToRemove.almostMatches(element)) {
-                            context.overlappedElements().remove(elementToRemove);
-                            anyMatch = true;
-                            break;
-                        }
-                    }
-                    if (!anyMatch) {
-                        writer.writeElement(element);
-                    } else if (element.getType() == 3 && element.hasTextMatrix()) {
-                        /*
-                        PDFTron Element with type "text" refers to a Tj command. If a Tm command is just above it in the pdf file, PDFTron will join the two commands and treat them as one Element.
-                        hasTextMatrix() checks for this case specifically. Also, Tm changes the position for a whole BT/ET segment, possibly containing multiple Tj commands.
-                        Therefore, the position of a following Tj is affected by not writing the first Element.
-                        This is why, we write only the Tm command:
-                        */
-                        writer.writeGStateChanges(element);
-                    }
-                }
-                default -> writer.writeElement(element);
-            }
-        }
-    }
-
-
-    private void processFormOverlappedElements(ElementWriter writer, Element formElement, InvisibleElementRemovalContext context) throws PDFNetException {
-
-        writer.writeElement(formElement);
-        Obj formObj = formElement.getXObject();
-
-        if (!context.visitedXObjIds().contains(formObj.getObjNum())) {
-            context.visitedXObjIds().add(formObj.getObjNum());
-            // writer needs to be newly initialized when entering a new content stream
-            // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
-            ElementWriter formWriter = new ElementWriter();
-            context.reader().formBegin();
-            formWriter.begin(formObj);
-
-            context.reader().clearChangeList();
-            formWriter.setDefaultGState(context.reader());
-
-            processOverlappedElements(formWriter, context);
-            formWriter.end();
-            formWriter.destroy();
-            context.reader().end();
-        }
-    }
-
-
-    private boolean isTextRenderedVisibly(GState gState) throws PDFNetException {
-
-        return gState.getTextRenderMode() != GState.e_invisible_text && //
-                !(gState.getTextRenderMode() == GState.e_fill_text && gState.getFillOpacity() == 0) && //
-                !(gState.getTextRenderMode() == GState.e_stroke_text && gState.getStrokeOpacity() == 0) && //
-                !(gState.getTextRenderMode() == GState.e_fill_stroke_text && gState.getFillOpacity() == 0 && gState.getStrokeOpacity() == 0);
-    }
-
-
-    private GeneralPath convertToGeneralPath(PathData pathData) throws PDFNetException {
-
-        GeneralPath linePath = new GeneralPath();
-        Iterator<Double> points = Doubles.asList(pathData.getPoints()).iterator();
-        Iterable<Byte> operators = Bytes.asList(pathData.getOperators());
-        for (var operator : operators) {
-            switch (operator) {
-                case PathData.e_moveto -> linePath.moveTo(points.next(), points.next());
-                case PathData.e_lineto -> linePath.lineTo(points.next(), points.next());
-                case PathData.e_cubicto -> linePath.curveTo(points.next(), points.next(), points.next(), points.next(), points.next(), points.next());
-                case PathData.e_closepath -> linePath.closePath();
-                case PathData.e_rect -> {
-                    double x = points.next();
-                    double y = points.next();
-                    double w = points.next();
-                    double h = points.next();
-                    linePath.moveTo(x, y);
-                    linePath.lineTo(x + w, y);
-                    linePath.lineTo(x + w, y + h);
-                    linePath.lineTo(x, y + h);
-                    linePath.closePath();
-                }
-                default -> throw new PDFNetException("Invalid Element Type", 0, "", "", "");
-            }
-        }
-        return linePath;
-    }
-
-
-    private boolean almostContains(Shape outer, Rectangle2D inner) {
-        //To address inconsistencies in the calculation of the bounding box we slightly shrink the inner rectangle
-
-        double x_with_tolerance = inner.getX() >= 0 ? inner.getX() + TOLERANCE : inner.getX() - TOLERANCE;
-        double y_with_tolerance = inner.getY() >= 0 ? inner.getY() + TOLERANCE : inner.getY() - TOLERANCE;
-        double height_with_tolerance = inner.getHeight() - (2 * TOLERANCE);
-        double width_with_tolerance = inner.getWidth() - (2 * TOLERANCE);
-        Rectangle2D innerRect = new Rectangle2D.Double(x_with_tolerance, y_with_tolerance, width_with_tolerance, height_with_tolerance);
-
-        return outer.contains(innerRect);
-    }
-
-
-    private boolean isFilledAndNonTransparent(Element element) throws PDFNetException {
-
-        return element.isFilled() && element.getGState().getFillOpacity() == 1;
-    }
-
-
-    @SneakyThrows
-    private void drawBBox(ElementWriter writer, Rectangle2D r, String hexcolor) {
-
-        ColorPt colorPt = new ColorPt(Integer.valueOf(hexcolor.substring(1, 3), 16) / 255d,
-                Integer.valueOf(hexcolor.substring(3, 5), 16) / 255d,
-                Integer.valueOf(hexcolor.substring(5, 7), 16) / 255d);
-        ElementBuilder eb = new ElementBuilder();
-        Element rect = eb.createRect(r.getX(), r.getY(), r.getWidth(), r.getHeight());
-        rect.setPathStroke(true);
-        rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
-        rect.getGState().setStrokeColor(colorPt);
-        writer.writePlacedElement(rect);
-
-        colorPt.destroy();
-        eb.destroy();
-    }
-
-
-    private static AffineTransform toAffineTransform(Matrix2D ctm) throws PDFNetException {
-
-        return new AffineTransform(ctm.getA(), ctm.getB(), ctm.getC(), ctm.getD(), ctm.getH(), ctm.getV());
-    }
-
-
-    @Builder
-    private record InvisibleElementRemovalContext(
-            boolean delta,
-            ElementReader reader,
-            ClippingPathStack clippingPathStack,
-            List<ElementFeatures> overlappedElements,
-            List<ElementFeatures> visibleElements,
-            Set<Long> visitedXObjIds) {
-
-    }
-
-}
@@ -14,6 +14,7 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
 import com.iqser.red.service.ocr.v1.api.model.OCRStatusUpdateResponse;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
@@ -68,14 +69,7 @@ public class OCRService {

        try (ByteArrayOutputStream transferOutputStream = new ByteArrayOutputStream()) {
            try (InputStream fileStream = fileStorageService.getOriginalFileAsStream(dossierId, fileId)) {
-                long removalStart = System.currentTimeMillis();
-                log.debug("Start invisible element removal for file with dossierId {} and fileId {}", dossierId, fileId);
                invisibleElementRemovalService.removeInvisibleElements(fileStream, transferOutputStream, false);
-                long removalEnd = System.currentTimeMillis();
-                log.debug("Invisible element removal successful for file with dossierId {} and fileId {}, took {}s",
-                        dossierId,
-                        fileId,
-                        format("%.1f", (removalEnd - removalStart) / 1000.0));
            }
            try (InputStream transferInputStream = new ByteArrayInputStream(transferOutputStream.toByteArray())) {
                long ocrStart = System.currentTimeMillis();
@@ -1,15 +1,20 @@
 package com.iqser.red.service.ocr.v1.server.service;

+import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
+import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.ocr.v1.api.model.DocumentRequest;
 import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
-import com.iqser.red.service.ocr.v1.server.initializer.PDFNetInitializer;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
+
 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
+import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.http.HttpStatus;
@@ -18,6 +23,9 @@ import org.springframework.stereotype.Service;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;

@Slf4j
@Service
@@ -33,40 +41,50 @@ public class OcrMessageReceiver {

    @RabbitHandler
    @RabbitListener(queues = MessagingConfiguration.OCR_QUEUE, concurrency = "1")
-    public void receiveOcr(String in) throws JsonProcessingException {
+    public void receiveOcr(Message in) throws JsonProcessingException {

-        DocumentRequest ocrRequestMessage = objectMapper.readValue(in, DocumentRequest.class);
+        DocumentRequest ocrRequestMessage = objectMapper.readValue(new String(in.getBody(), StandardCharsets.UTF_8), DocumentRequest.class);

        log.info("Start ocr for file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());

-        setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+        try {
+            setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());

-        if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) {
-            byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
-            fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile);
-        }
-
-        try (var transferStream = new ByteArrayOutputStream()) {
-            ocrService.runOcrOnDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), transferStream);
-            try (var inputStream = new ByteArrayInputStream(transferStream.toByteArray())) {
-                fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), inputStream);
+            if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) {
+                byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+                fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile);
            }
-        } catch (IOException e) {
-            log.error("Failed to store file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
-            throw new RuntimeException(e);
-        }

-        fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+            try (var transferStream = new ByteArrayOutputStream()) {
+                ocrService.runOcrOnDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), transferStream);
+                try (var inputStream = new ByteArrayInputStream(transferStream.toByteArray())) {
+                    fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), inputStream);
+                }
+            } catch (IOException e) {
+                log.error("Failed to store file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+                throw new RuntimeException(e);
+            }
+
+            fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+        } catch (Exception e) {
+            log.warn("An exception occurred in ocr file stage: {}", e.getMessage() );
+            in.getMessageProperties().getHeaders().put(X_ERROR_INFO_HEADER, e.getMessage());
+            in.getMessageProperties().getHeaders().put(X_ERROR_INFO_TIMESTAMP_HEADER, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+            throw e;
+        }
    }


    @RabbitHandler
    @RabbitListener(queues = MessagingConfiguration.OCR_DLQ, concurrency = "1")
-    public void receiveOcrDQL(String in) throws JsonProcessingException {
+    public void receiveOcrDQL(Message failedMessage) throws JsonProcessingException {

-        DocumentRequest ocrRequestMessage = objectMapper.readValue(in, DocumentRequest.class);
+        DocumentRequest ocrRequestMessage = objectMapper.readValue(new String(failedMessage.getBody(), StandardCharsets.UTF_8), DocumentRequest.class);
        log.info("OCR DQL received: {}", ocrRequestMessage);
-        fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
+        String errorMessage = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
+        OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
+        timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
+        fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp));
    }


@@ -9,24 +9,30 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;

+import com.iqser.red.service.ocr.v1.server.initializer.PDFNetInitializer;
 import com.iqser.red.service.ocr.v1.server.utils.FileSystemBackedStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.pdftron.pdf.PDFNet;

+import jakarta.annotation.PostConstruct;
 import lombok.SneakyThrows;

@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(AbstractTest.TestConfiguration.class)
+@Import({AbstractTest.TestConfiguration.class, PDFNetInitializer.class})
+@AutoConfigureObservability
 public class AbstractTest {

    @Autowired
@@ -35,6 +41,14 @@ public class AbstractTest {
    @MockBean
    protected RabbitTemplate rabbitTemplate;

+    private static String pdftronLicense;
+
+    @PostConstruct
+    @SneakyThrows
+    public void init() {
+        PDFNet.initialize(pdftronLicense);
+    }
+

    @AfterAll
    public static void terminatePDFNet() {
@@ -62,7 +76,8 @@ public class AbstractTest {


    @Configuration
-    @EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
    public static class TestConfiguration {

        @Bean
@@ -1,7 +1,7 @@
 package com.iqser.red.service.ocr.v1.server;

+import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
-import static com.iqser.red.service.ocr.v1.server.utils.PdfTextExtraction.extractAllTextFromDocument;
 import static org.assertj.core.api.Assertions.assertThat;

 import java.io.FileInputStream;
@@ -1,7 +1,7 @@
 package com.iqser.red.service.ocr.v1.server.service;

+import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
-import static com.iqser.red.service.ocr.v1.server.utils.PdfTextExtraction.extractAllTextFromDocument;
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;

 import java.io.FileInputStream;
@@ -9,16 +9,18 @@ import java.io.FileOutputStream;

 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;

+import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
 import com.iqser.red.service.ocr.v1.server.AbstractTest;

 import lombok.SneakyThrows;

 public class InvisibleElementRemovalServiceTest extends AbstractTest {

-    @Autowired
-    private InvisibleElementRemovalService invisibleElementRemovalService;
+   @Autowired
+   private InvisibleElementRemovalService invisibleElementRemovalService;


    @Test
@@ -44,5 +46,6 @@ public class InvisibleElementRemovalServiceTest extends AbstractTest {
            String[] text = extractAllTextFromDocument(fileStream).split("\n");
            assertThat(text).contains("APPENDIX 16 Pathology Report", "Amendment 1", "Page 255 of 260");
        }
+
    }
 }
@@ -1,35 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.utils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.pdftron.common.PDFNetException;
-import com.pdftron.pdf.PDFDoc;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.PageIterator;
-import com.pdftron.pdf.TextExtractor;
-
-
-public class PdfTextExtraction {
-
-    public static String extractAllTextFromDocument(InputStream fileStream) throws IOException, PDFNetException {
-
-        PDFDoc pdfDoc = new PDFDoc(fileStream);
-        TextExtractor extractor = new TextExtractor();
-        List<String> texts = new ArrayList<>();
-
-        PageIterator iterator = pdfDoc.getPageIterator();
-        while (iterator.hasNext()) {
-            Page page = iterator.next();
-            extractor.begin(page);
-            texts.add(extractor.getAsText());
-        }
-
-        extractor.destroy();
-        pdfDoc.close();
-        return String.join("\n", texts);
-    }
-
-}
@@ -7,13 +7,13 @@
    <parent>
        <groupId>com.iqser.red</groupId>
        <artifactId>platform-dependency</artifactId>
-        <version>1.17.0</version>
+        <version>2.2.0</version>
        <relativePath/>
    </parent>

    <groupId>com.iqser.red.service</groupId>
    <artifactId>ocr-service-v1</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>3.0-SNAPSHOT</version>

    <packaging>pom</packaging>

@@ -27,7 +27,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.22.0</version>
+                <version>2.1.0</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -39,12 +39,12 @@
                <plugin>
                    <groupId>org.sonarsource.scanner.maven</groupId>
                    <artifactId>sonar-maven-plugin</artifactId>
-                    <version>3.9.0.2155</version>
+                    <version>3.9.1.2184</version>
                </plugin>
                <plugin>
                    <groupId>org.owasp</groupId>
                    <artifactId>dependency-check-maven</artifactId>
-                    <version>6.3.1</version>
+                    <version>8.1.2</version>
                    <configuration>
                        <format>ALL</format>
                    </configuration>
@@ -6,12 +6,11 @@

    <groupId>com.iqser.red.service</groupId>
    <artifactId>ocr-service</artifactId>
-    <version>2.0-SNAPSHOT</version>
+    <version>3.0-SNAPSHOT</version>

    <packaging>pom</packaging>

    <modules>
-        <module>bamboo-specs</module>
        <module>ocr-service-v1</module>
        <module>ocr-service-image-v1</module>
    </modules>
Author	SHA1	Message	Date
Dominique Eifländer	7dcc59c750	Merge branch 'RED-6072-timestamp' into 'master' RED-6072 - As Operation I want to see why files are in an ERROR state Closes RED-6072 See merge request redactmanager/ocr-service!2	2023-06-15 14:08:39 +02:00
Corina Olariu	b15bcfa372	RED-6072 - As Operation I want to see why files are in an ERROR state - add timestamp to error info	2023-06-15 14:37:32 +03:00
Christoph Schabert	d2f38d7cf0	Update 2 files - /.gitlab-ci.yml - /pom.xml	2023-05-31 15:44:02 +02:00
Christoph Schabert	df59fd83ed	Update 6 files - /bamboo-specs/src/main/java/buildjob/PlanSpec.java - /bamboo-specs/src/main/resources/scripts/build-java.sh - /bamboo-specs/src/main/resources/scripts/sonar-java.sh - /bamboo-specs/src/test/java/buildjob/PlanSpecTest.java - /bamboo-specs/pom.xml - /.gitlab-ci.yml	2023-05-31 15:10:37 +02:00
Corina Olariu	e400a04644	Pull request #22 : RED-6072 Merge in RED/ocr-service from RED-6072 to master * commit '961129b917e92b4dda2085215d3f15e205ec35a3': RED-6072 - As Operation I want to see why files are in an ERROR state - update the ocr failed with error information - in case of exception the reason is caught and added tot the message received from the queue RED-6072 - As Operation I want to see why files are in an ERROR state update from iqser to knecon	2023-05-25 14:59:04 +02:00
Dominique Eiflaender	933b4bd3de	Pull request #23 : RED-5567: Upgraded to lastest pdftron version Merge in RED/ocr-service from RED-5567-exptest to master * commit 'ee94f1c711c923e865e559499674c1658c87fbb4': RED-5567: Upgraded to lastest pdftron version	2023-05-25 14:19:18 +02:00
deiflaender	ee94f1c711	RED-5567: Upgraded to lastest pdftron version	2023-05-25 14:14:59 +02:00
devplant	961129b917	RED-6072 - As Operation I want to see why files are in an ERROR state - update the ocr failed with error information - in case of exception the reason is caught and added tot the message received from the queue	2023-05-25 14:29:40 +03:00
devplant	0307da9093	RED-6072 - As Operation I want to see why files are in an ERROR state update from iqser to knecon	2023-05-25 14:20:14 +03:00
Corina Olariu	aac56cc2ed	Pull request #21 : RED-5694 - Upgrade spring-boot to 3.0 Merge in RED/ocr-service from RED-5694-storage to master * commit '828b4f53c837a77b860dc3df20a8e65b1330eaa8': RED-5694 - Upgrade spring-boot to 3.0 - add -DknownExploitedEnabled=false - import StorageAutoConfiguration to Application	2023-04-21 14:08:04 +02:00
devplant	828b4f53c8	RED-5694 - Upgrade spring-boot to 3.0 - add -DknownExploitedEnabled=false - import StorageAutoConfiguration to Application	2023-04-20 16:14:28 +03:00
Corina Olariu	062cf46e30	Pull request #20 : RED-5694 Merge in RED/ocr-service from RED-5694 to master * commit 'f72001c4ac0431c482cc467e948bdc972643d785': RED-5694 - Upgrade spring-boot to 3.0 - remove commented code RED-5694 - Upgrade spring-boot to 3.0 - remove versions already defined in spring boot RED-5694 - Upgrade spring-boot to 3.0 - remove unused import RED-5694 - Upgrade spring-boot to 3.0 - update platform-dependency - update other dependencies versions to the latest	2023-04-06 12:42:23 +02:00
devplant	f72001c4ac	RED-5694 - Upgrade spring-boot to 3.0 - remove commented code	2023-04-06 13:36:43 +03:00
devplant	81fd35f9c0	RED-5694 - Upgrade spring-boot to 3.0 - remove versions already defined in spring boot	2023-04-06 13:35:42 +03:00
devplant	17b1932926	RED-5694 - Upgrade spring-boot to 3.0 - remove unused import	2023-04-06 11:57:36 +03:00
devplant	c00e9290bc	RED-5694 - Upgrade spring-boot to 3.0 - update platform-dependency - update other dependencies versions to the latest	2023-04-06 11:52:44 +03:00
Thomas Beyer	d9e4f79099	Pull request #19 : RED-4875 1 Merge in RED/ocr-service from RED-4875_1 to master * commit '036203c24a1f5eb1588e945eb020a666cac5dba2': RED-4875 - delete commented out classes RED-4875 - set version of pdftron-common-logics to newest (release) RED-4875 - removed duration log RED-4875 - update version of pdftron-commons to newest RED-4875 - update version of pdftron logic commons to newest RED-4875 - set version of common pdftron logics to newest and move PdfTextExtraction to this new repo RED-4875 - update version of pdftron-logic-commons to newest RED-4875 - call logic of new repo pdftron-logic-commons instead of local one	2023-03-22 12:07:24 +01:00
Thomas Beyer	036203c24a	RED-4875 - delete commented out classes	2023-03-21 18:20:26 +01:00
Thomas Beyer	6ac71f6d94	RED-4875 - set version of pdftron-common-logics to newest (release)	2023-03-21 17:00:41 +01:00
Thomas Beyer	dd12611fdc	RED-4875 - removed duration log	2023-03-21 13:18:41 +01:00
Thomas Beyer	b0c4c25bec	RED-4875 - update version of pdftron-commons to newest	2023-03-21 12:28:57 +01:00
Thomas Beyer	5efa0e96a8	RED-4875 - update version of pdftron logic commons to newest	2023-03-20 11:25:38 +01:00
Thomas Beyer	fd92419895	RED-4875 - set version of common pdftron logics to newest and move PdfTextExtraction to this new repo	2023-03-20 10:01:33 +01:00
Thomas Beyer	142e8cf957	RED-4875 - update version of pdftron-logic-commons to newest	2023-03-17 17:25:52 +01:00
Thomas Beyer	143538fa40	RED-4875 - call logic of new repo pdftron-logic-commons instead of local one	2023-03-17 10:33:48 +01:00
Christoph Schabert	74a094b42d	Change Tagging	2023-03-13 09:41:59 +01:00
Timo Bejan	8ab4092046	Pull request #18 : RED-6162 - bumped version Merge in RED/ocr-service from RED-6162 to master * commit 'c86cabf70a9ec90687c3f123768f2293da49c4ff': RED-6162 - bumped version RED-6162 - bumped version	2023-03-10 21:45:38 +01:00
Timo Bejan	c86cabf70a	RED-6162 - bumped version	2023-03-10 22:43:35 +02:00
Dominique Eiflaender	936a795797	Pull request #17 : RED-4645: Multitenancy for storage Merge in RED/ocr-service from RED-4645 to master * commit 'e5df0ec65864667134affad2af2a5118126355ce': RED-4645: Multitenancy for storage	2023-03-10 16:42:43 +01:00
Timo Bejan	0d9365d020	RED-6162 - bumped version	2023-03-10 15:59:25 +02:00