RED-6126: In the OCRService, OCR Text is not applied to Document

*removed init/terminate calls again
*added destroy()/close() at every opportunity

ocr-service-image-v1/src/main/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM red/base-image:2.0.2
+FROM red/base-image:2.0.0
 
 COPY "libs/pdftron/OCRModuleLinux.tar.gz" .
 RUN tar xvzf OCRModuleLinux.tar.gz

ocr-service-v1/ocr-service-api-v1/pom.xml

@@ -14,7 +14,8 @@
     <version>1.0-SNAPSHOT</version>
 
     <properties>
-        <persistence-service.version>2.1.0</persistence-service.version>
+        <persistence-service.version>1.269.0</persistence-service.version>
+        <redaction-service.version>3.155.0</redaction-service.version>
         <dsljson.version>1.9.9</dsljson.version>
     </properties>
 
@@ -42,9 +43,26 @@
 
         <dependency>
             <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-internal-api-v1</artifactId>
+            <artifactId>persistence-service-api-v1</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>redaction-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
             <version>${persistence-service.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>redaction-service-api-v1</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>persistence-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
+            <version>${redaction-service.version}</version>
+        </dependency>
 
     </dependencies>
 </project>

ocr-service-v1/ocr-service-api-v1/src/test/java/com/iqser/red/service/ocr/v1/api/model/IdentityTest.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.ocr.v1.api.model;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import org.junit.jupiter.api.Test;
-
-public class IdentityTest {
-
-    @Test
-    public void mockTest() {
-
-        int i = 1;
-        assertThat(i).isEqualTo(1);
-    }
-
-}
-

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/Application.java

@@ -10,11 +10,9 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.annotation.EnableAsync;
 
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
-import com.iqser.red.service.ocr.v1.server.multitenancy.AsyncConfig;
-import com.iqser.red.service.ocr.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
-import com.iqser.red.service.ocr.v1.server.multitenancy.MultiTenancyWebConfiguration;
 import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
 
 import io.micrometer.core.aop.TimedAspect;
@@ -23,7 +21,7 @@ import io.micrometer.core.instrument.MeterRegistry;
 @EnableAsync
 @EnableConfigurationProperties(OcrServiceSettings.class)
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
-@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class})
+@Import({DefaultWebMvcConfiguration.class, MessagingConfiguration.class})
 @EnableFeignClients(basePackageClasses = FileStatusProcessingUpdateClient.class)
 public class Application {
 

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/client/FileStatusProcessingUpdateClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.ocr.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;
+import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
 
 @FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
 public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/client/TenantsClient.java

@@ -1,10 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.client;
-
-import org.springframework.cloud.openfeign.FeignClient;
-
-import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
-
-@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
-public interface TenantsClient extends TenantsResource {
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/model/ImagePosition.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.ocr.v1.server.model;
 
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/AsyncConfig.java

@@ -1,27 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import java.util.concurrent.Executor;
-
-import org.springframework.context.annotation.Configuration;
-import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
-import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
-
-@Configuration
-public class AsyncConfig extends AsyncConfigurerSupport {
-
-    @Override
-    public Executor getAsyncExecutor() {
-
-        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
-
-        executor.setCorePoolSize(7);
-        executor.setMaxPoolSize(42);
-        executor.setQueueCapacity(11);
-        executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
-        executor.setTaskDecorator(new TenantAwareTaskDecorator());
-        executor.initialize();
-
-        return executor;
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/EncryptionDecryptionService.java

@@ -1,105 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
-import java.security.spec.KeySpec;
-import java.util.Base64;
-
-import javax.annotation.PostConstruct;
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.SecretKeyFactory;
-import javax.crypto.spec.GCMParameterSpec;
-import javax.crypto.spec.PBEKeySpec;
-import javax.crypto.spec.SecretKeySpec;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import lombok.SneakyThrows;
-
-@Service
-public class EncryptionDecryptionService {
-
-    @Value("${ocr-service.crypto.key:redaction}")
-    private String key;
-
-    private SecretKey secretKey;
-    private byte[] iv;
-
-
-    @SneakyThrows
-    @PostConstruct
-    protected void postConstruct() {
-
-        SecureRandom secureRandom = new SecureRandom();
-        iv = new byte[12];
-        secureRandom.nextBytes(iv);
-        secretKey = generateSecretKey(key, iv);
-    }
-
-
-    @SneakyThrows
-    public String encrypt(String strToEncrypt) {
-
-        return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
-    }
-
-
-    @SneakyThrows
-    public String decrypt(String strToDecrypt) {
-
-        byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
-        return new String(decrypt(bytes), StandardCharsets.UTF_8);
-    }
-
-
-    @SneakyThrows
-    public byte[] encrypt(byte[] data) {
-
-        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
-        GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
-        cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
-        byte[] encryptedData = cipher.doFinal(data);
-        ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
-        byteBuffer.putInt(iv.length);
-        byteBuffer.put(iv);
-        byteBuffer.put(encryptedData);
-        return byteBuffer.array();
-    }
-
-
-    @SneakyThrows
-    public byte[] decrypt(byte[] encryptedData) {
-
-        ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
-        int noonceSize = byteBuffer.getInt();
-        if (noonceSize < 12 || noonceSize >= 16) {
-            throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
-        }
-        byte[] iv = new byte[noonceSize];
-        byteBuffer.get(iv);
-
-        SecretKey secretKey = generateSecretKey(key, iv);
-
-        byte[] cipherBytes = new byte[byteBuffer.remaining()];
-        byteBuffer.get(cipherBytes);
-
-        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
-        GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
-        cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
-        return cipher.doFinal(cipherBytes);
-    }
-
-
-    @SneakyThrows
-    public SecretKey generateSecretKey(String password, byte[] iv) {
-
-        KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
-        SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
-        byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
-        return new SecretKeySpec(key, "AES");
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/ForwardTenantInterceptor.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import org.springframework.stereotype.Component;
-
-import feign.RequestInterceptor;
-import feign.RequestTemplate;
-
-@Component
-public class ForwardTenantInterceptor implements RequestInterceptor {
-
-    public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
-
-    @Override
-    public void apply(RequestTemplate template) {
-        template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
-    }
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java

@@ -1,48 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import static com.iqser.red.service.ocr.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME;
-
-import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.BeansException;
-import org.springframework.beans.factory.config.BeanPostProcessor;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-public class MultiTenancyMessagingConfiguration {
-
-    @Bean
-    public static BeanPostProcessor multitenancyBeanPostProcessor() {
-
-        return new BeanPostProcessor() {
-
-            @Override
-            public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
-
-                if (bean instanceof RabbitTemplate) {
-
-                    ((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
-                        m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId());
-                        return m;
-                    });
-
-                } else if (bean instanceof AbstractRabbitListenerContainerFactory) {
-
-                    ((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
-                        String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME);
-
-                        if (tenant != null) {
-                            TenantContext.setTenantId(tenant);
-                        } else {
-                            throw new RuntimeException("No Tenant is set queue message");
-                        }
-                        return m;
-                    });
-                }
-                return bean;
-            }
-        };
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/MultiTenancyWebConfiguration.java

@@ -1,28 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
-
-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-
-@Configuration
-public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
-
-    private final TenantInterceptor tenantInterceptor;
-
-
-    @Autowired
-    public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
-
-        this.tenantInterceptor = tenantInterceptor;
-    }
-
-
-    @Override
-    public void addInterceptors(InterceptorRegistry registry) {
-
-        registry.addWebRequestInterceptor(tenantInterceptor);
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/StorageConnectionProviderImpl.java

@@ -1,45 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.ocr.v1.server.client.TenantsClient;
-import com.iqser.red.storage.commons.model.AzureStorageConnection;
-import com.iqser.red.storage.commons.model.S3StorageConnection;
-import com.iqser.red.storage.commons.service.StorageConnectionProvider;
-
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class StorageConnectionProviderImpl implements StorageConnectionProvider {
-
-    private final TenantsClient tenantsClient;
-    private final EncryptionDecryptionService encryptionDecryptionService;
-
-
-    @Override
-    public AzureStorageConnection getAzureStorageConnection(String tenantId) {
-
-        var tenant = tenantsClient.getTenant(tenantId);
-        return AzureStorageConnection.builder()
-                .connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
-                .containerName(tenant.getAzureStorageConnection().getContainerName())
-                .build();
-    }
-
-
-    @Override
-    public S3StorageConnection getS3StorageConnection(String tenantId) {
-
-        var tenant = tenantsClient.getTenant(tenantId);
-        return S3StorageConnection.builder()
-                .key(tenant.getS3StorageConnection().getKey())
-                .secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
-                .signerType(tenant.getS3StorageConnection().getSignerType())
-                .bucketName(tenant.getS3StorageConnection().getBucketName())
-                .region(tenant.getS3StorageConnection().getRegion())
-                .endpoint(tenant.getS3StorageConnection().getEndpoint())
-                .build();
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/TenantAwareTaskDecorator.java

@@ -1,23 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import org.springframework.core.task.TaskDecorator;
-import org.springframework.lang.NonNull;
-
-public class TenantAwareTaskDecorator implements TaskDecorator {
-
-    @Override
-    @NonNull
-    public Runnable decorate(@NonNull Runnable runnable) {
-
-        String tenantId = TenantContext.getTenantId();
-        return () -> {
-            try {
-                TenantContext.setTenantId(tenantId);
-                runnable.run();
-            } finally {
-                TenantContext.setTenantId(null);
-            }
-        };
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/TenantContext.java

@@ -1,29 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public final class TenantContext {
-
-    private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
-
-
-    public static void setTenantId(String tenantId) {
-
-        log.debug("Setting tenantId to " + tenantId);
-        currentTenant.set(tenantId);
-    }
-
-
-    public static String getTenantId() {
-
-        return currentTenant.get();
-    }
-
-
-    public static void clear() {
-
-        currentTenant.remove();
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/multitenancy/TenantInterceptor.java

@@ -1,35 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.multitenancy;
-
-import org.springframework.stereotype.Component;
-import org.springframework.ui.ModelMap;
-import org.springframework.web.context.request.WebRequest;
-import org.springframework.web.context.request.WebRequestInterceptor;
-
-@Component
-public class TenantInterceptor implements WebRequestInterceptor {
-
-    public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
-
-
-    @Override
-    public void preHandle(WebRequest request) {
-
-        if (request.getHeader(TENANT_HEADER_NAME) != null) {
-            TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
-        }
-    }
-
-
-    @Override
-    public void postHandle(WebRequest request, ModelMap model) {
-
-        TenantContext.clear();
-    }
-
-
-    @Override
-    public void afterCompletion(WebRequest request, Exception ex) {
-
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/DataExtractionService.java

@@ -0,0 +1,2 @@
+package com.iqser.red.service.ocr.v1.server.service;public class DataExtractionService {
+}

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/FileStorageService.java

@@ -7,8 +7,7 @@ import org.apache.commons.io.IOUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.ocr.v1.server.model.image.ImageServiceResponse;
-import com.iqser.red.service.ocr.v1.server.multitenancy.TenantContext;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.storage.commons.service.StorageService;
 
 import lombok.RequiredArgsConstructor;
@@ -32,39 +31,39 @@ public class FileStorageService {
     @SneakyThrows
     public byte[] getOriginalFile(String dossierId, String fileId) {
 
-        return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
+        return IOUtils.toByteArray(storageService.getObject(getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
     }
 
 
     @SneakyThrows
     public InputStream getOriginalFileAsStream(String dossierId, String fileId) {
 
-        return storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
+        return storageService.getObject(getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
     }
 
 
     public void storeOriginalFile(String dossierId, String fileId, InputStream stream) {
 
-        storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
+        storageService.storeObject(getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
     }
 
 
     public boolean untouchedFileExists(String dossierId, String fileId) {
 
-        return storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED));
+        return storageService.objectExists(getStorageId(dossierId, fileId, FileType.UNTOUCHED));
     }
 
 
     public void storeUntouchedFile(String dossierId, String fileId, byte[] data) {
 
-        storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
+        storageService.storeObject(getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
     }
 
 
     @SneakyThrows
     public ImageServiceResponse getImageServiceResponse(String dossierId, String fileId) {
 
-        return storageService.readJSONObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
+        return storageService.readJSONObject(getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
     }
 
 }

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/InvisibleElementRemovalService.java

@@ -87,13 +87,17 @@ public class InvisibleElementRemovalService {
                     .visitedXObjIds(visitedXObjIds)
                     .build();
 
+
             removeClippedElementsAndInvisibleTextAndRememberOverlappedElements(page, writer, context);
 
             context.visitedXObjIds().clear();
 
             removeOverlappedElements(page, writer, context);
+            reader.end();
+            writer.end();
         }
 
+
         try {
             pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
         } catch (Exception e) {
@@ -226,7 +230,6 @@ public class InvisibleElementRemovalService {
             formWriter.setDefaultGState(context.reader());
 
             processElements(formWriter, context);
-            formWriter.end();
             formWriter.destroy();
             context.reader().end();
         }
@@ -234,7 +237,6 @@ public class InvisibleElementRemovalService {
 
 
     private void processPath(Element pathElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
-
         PathData pathData = pathElement.getPathData();
 
         if (pathData.getOperators().length == 0 && pathData.getPoints().length == 0) {
@@ -364,7 +366,6 @@ public class InvisibleElementRemovalService {
             formWriter.setDefaultGState(context.reader());
 
             processOverlappedElements(formWriter, context);
-            formWriter.end();
             formWriter.destroy();
             context.reader().end();
         }
@@ -389,7 +390,8 @@ public class InvisibleElementRemovalService {
             switch (operator) {
                 case PathData.e_moveto -> linePath.moveTo(points.next(), points.next());
                 case PathData.e_lineto -> linePath.lineTo(points.next(), points.next());
-                case PathData.e_cubicto -> linePath.curveTo(points.next(), points.next(), points.next(), points.next(), points.next(), points.next());
+                case PathData.e_cubicto ->
+                        linePath.curveTo(points.next(), points.next(), points.next(), points.next(), points.next(), points.next());
                 case PathData.e_closepath -> linePath.closePath();
                 case PathData.e_rect -> {
                     double x = points.next();

ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/OCRService.java

@@ -1,35 +1,23 @@
 package com.iqser.red.service.ocr.v1.server.service;
 
-import static java.lang.String.format;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.ocr.v1.api.model.OCRStatusUpdateResponse;
 import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
 import com.pdftron.common.PDFNetException;
-import com.pdftron.pdf.OCRModule;
-import com.pdftron.pdf.OCROptions;
-import com.pdftron.pdf.Optimizer;
-import com.pdftron.pdf.PDFDoc;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.RectCollection;
+import com.pdftron.pdf.*;
 import com.pdftron.sdf.SDFDoc;
-
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import java.io.*;
+import java.util.Map;
+
+import static java.lang.String.format;
 
 @Slf4j
 @Service
@@ -72,10 +60,7 @@ public class OCRService {
                 log.debug("Start invisible element removal for file with dossierId {} and fileId {}", dossierId, fileId);
                 invisibleElementRemovalService.removeInvisibleElements(fileStream, transferOutputStream, false);
                 long removalEnd = System.currentTimeMillis();
-                log.debug("Invisible element removal successful for file with dossierId {} and fileId {}, took {}s",
-                        dossierId,
-                        fileId,
-                        format("%.1f", (removalEnd - removalStart) / 1000.0));
+                log.debug("Invisible element removal successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, format("%.1f", (removalEnd - removalStart) / 1000.0));
             }
             try (InputStream transferInputStream = new ByteArrayInputStream(transferOutputStream.toByteArray())) {
                 long ocrStart = System.currentTimeMillis();
@@ -93,22 +78,18 @@ public class OCRService {
         PDFDoc pdfDoc = new PDFDoc(fileStream);
 
         Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, true);
-        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
-                objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).build()));
 
         // Optimization:
         // When a page does not have a TextZone, PDFTron whites out the page. But, PDFTron scans it anyway, resulting in a longer runtime.
         // So, we need to remove pages without images.
         // Furthermore, creating a new document is *much* faster than reusing the same document and adding/removing pages one by one.
         // Therefore, we create a new Document with a single page for every page that contains an image.
-        // For some reason, if we insert the OCR singlePageDoc into the original PDFDoc inside the loop, for some documents, the RAM Usage increases exponentially with every page.
-        // This is why, we replace the OCRed Pages outside the main loop.
         int numProcessedPages = 0;
-        Map<Integer, PDFDoc> pageIdToSinglePagePdfDoc = new HashMap<>();
         for (Integer pageId : pageIdToRectCollection.keySet()) {
             try {
                 PDFDoc singlePagePdfDoc = extractSinglePagePdfDoc(pdfDoc, pageId);
                 processOcr(pageIdToRectCollection, pageId, singlePagePdfDoc);
+                ++numProcessedPages;
 
                 log.info("{}/{} Page {} done, OCR regions {}",
                         numProcessedPages,
@@ -116,8 +97,9 @@ public class OCRService {
                         pageId,
                         getAllOcrTextZonesAsString(pageIdToRectCollection, pageId));
 
-                pageIdToSinglePagePdfDoc.put(pageId, singlePagePdfDoc);
-                ++numProcessedPages;
+                replaceOriginalPageWithOcrPage(pdfDoc, pageId, singlePagePdfDoc);
+
+                singlePagePdfDoc.close();
 
                 rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
                         objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder()
@@ -131,9 +113,6 @@ public class OCRService {
                 throw new RuntimeException(e);
             }
         }
-        log.info("Copying {} OCRed Pages into original Document", pageIdToSinglePagePdfDoc.size());
-        pageIdToSinglePagePdfDoc.forEach((pageId, singlePagePdfDoc) -> replaceOriginalPageWithOcrPage(pdfDoc, pageId, singlePagePdfDoc));
-        Optimizer.optimize(pdfDoc);
 
         rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
                 objectMapper.writeValueAsString(OCRStatusUpdateResponse.builder()
@@ -143,6 +122,7 @@ public class OCRService {
                         .ocrFinished(true)
                         .build()));
 
+        Optimizer.optimize(pdfDoc);
         try {
             pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
         } catch (Exception e) {
@@ -174,13 +154,11 @@ public class OCRService {
     }
 
 
-    @SneakyThrows
-    private static void replaceOriginalPageWithOcrPage(PDFDoc pdfDoc, Integer pageId, PDFDoc singlePagePdfDoc) {
+    private static void replaceOriginalPageWithOcrPage(PDFDoc pdfDoc, Integer pageId, PDFDoc ocrPageDoc) throws PDFNetException {
 
-        Page ocrPage = singlePagePdfDoc.getPage(1);
+        Page ocrPage = ocrPageDoc.getPage(1);
         pdfDoc.pageInsert(pdfDoc.getPageIterator(pageId), ocrPage);
         pdfDoc.pageRemove(pdfDoc.getPageIterator(pageId + 1));
-        singlePagePdfDoc.close();
     }
 
 

ocr-service-v1/ocr-service-server-v1/src/main/resources/application.yml

@@ -40,6 +40,10 @@ management:
 
 
 storage:
+  signer-type: 'AWSS3V4SignerType'
+  bucket-name: 'redaction'
+  region: 'us-east-1'
+  endpoint: 'https://s3.amazonaws.com'
   backend: 's3'
 
 pdftron.license: ${PDFTRON_LICENSE}

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/OcrServiceIntegrationTest.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.ocr.v1.server;
 
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
-import static com.iqser.red.service.ocr.v1.server.utils.PdfTextExtraction.extractAllTextFromDocument;
+import static com.iqser.red.service.ocr.v1.server.utils.PdfUtils.extractAllTextFromDocument;
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.io.FileInputStream;
@@ -16,19 +16,16 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.core.io.ClassPathResource;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.ocr.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.ocr.v1.server.service.FileStorageService;
 import com.iqser.red.service.ocr.v1.server.service.OCRService;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.pdftron.pdf.OCRModule;
 
 import io.micrometer.prometheus.PrometheusMeterRegistry;
 import io.micrometer.prometheus.PrometheusTimer;
 import lombok.SneakyThrows;
 
-
-@Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
-@SpringBootTest(properties = {"pdftron.ocrmodule.path=/YourOCRModulePath"})
+@SpringBootTest(properties = {"pdftron.ocrmodule.path=/home/kschuettler/iqser/PDFTron/ocr/Lib/"})
 public class OcrServiceIntegrationTest extends AbstractTest {
 
     @Autowired
@@ -42,12 +39,13 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
     @BeforeEach
     @SneakyThrows
+    @Disabled
     public void assertOCRModuleIsLoaded() {
         assert OCRModule.isModuleAvailable();
     }
 
-
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testOCRMetrics() {
 
         testOCR("Watermark");
@@ -63,13 +61,38 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testOcr() {
         // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
         String text = testOCR("StitchedImagesMultiPage");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+        testOCR("131 IDD0000261725");
+
     }
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testManyRotatedImages() {
         // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
         String text = testOCR("manyRotatedImages");
@@ -78,6 +101,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testMergeImages() {
         // check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
         String text = testOCR("merge_images");
@@ -98,6 +122,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testOCRWatermark() {
 
         assertThat(testOCR("Watermark")).contains("syngenta");
@@ -105,6 +130,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
 
 
     @Test
+    @Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
     public void testOCRInvisibleText() {
 
         String text = testOCR("InvisibleText");
@@ -119,7 +145,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/" + fileName + ".pdf");
         var originId = FileStorageService.getStorageId("dossier", "file", FileType.ORIGIN);
         try (var fileStream = pdfFileResource.getInputStream()) {
-            storageService.storeObject(TenantContext.getTenantId(), originId, fileStream);
+            storageService.storeObject(originId, fileStream);
         }
         try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
             ocrService.runOcrOnDocument("dossier", "file", out);

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/service/DataExtractionServiceTest.java

@@ -0,0 +1,14 @@
+package com.iqser.red.service.ocr.v1.server.service;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import org.junit.jupiter.api.Test;
+
+class DataExtractionServiceTest {
+
+    @Test
+    void extractData() {
+
+    }
+
+}

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/service/ImagePositionRetrievalServiceTest.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.ocr.v1.server.service;
 
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
-import static com.iqser.red.service.ocr.v1.server.utils.PdfDraw.drawGrid;
-import static com.iqser.red.service.ocr.v1.server.utils.PdfDraw.drawRectCollection;
+import static com.iqser.red.service.ocr.v1.server.utils.PdfUtils.drawGrid;
+import static com.iqser.red.service.ocr.v1.server.utils.PdfUtils.drawRectCollection;
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.io.FileInputStream;

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/service/InvisibleElementRemovalServiceTest.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.ocr.v1.server.service;
 
 import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
-import static com.iqser.red.service.ocr.v1.server.utils.PdfTextExtraction.extractAllTextFromDocument;
+import static com.iqser.red.service.ocr.v1.server.utils.PdfUtils.extractAllTextFromDocument;
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 
 import java.io.FileInputStream;

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/utils/FileSystemBackedStorageService.java

@@ -32,7 +32,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @SneakyThrows
     @Override
-    public InputStreamResource getObject(String tenantId, String objectId) {
+    public InputStreamResource getObject(String objectId) {
 
         var res = dataMap.get(objectId);
         if (res == null) {
@@ -44,22 +44,28 @@ public class FileSystemBackedStorageService implements StorageService {
 
 
     @Override
-    public void deleteObject(String tenantId, String objectId) {
+    public void deleteObject(String objectId) {
 
         dataMap.remove(objectId);
     }
 
 
     @Override
-    public boolean objectExists(String tenantId, String objectId) {
+    public boolean objectExists(String objectId) {
 
         return dataMap.containsKey(objectId);
     }
 
 
+    @Override
+    public void init() {
+
+    }
+
+
     @Override
     @SneakyThrows
-    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
+    public <T> void storeJSONObject(String objectId, T any) {
 
         File tempFile = File.createTempFile("test", ".tmp");
         getMapper().writeValue(new FileOutputStream(tempFile), any);
@@ -75,7 +81,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @Override
     @SneakyThrows
-    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
+    public <T> T readJSONObject(String objectId, Class<T> clazz) {
 
         if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
             throw new StorageObjectDoesNotExist("Stored object not found");
@@ -98,7 +104,7 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @Override
     @SneakyThrows
-    public void storeObject(String tenantId, String objectId, InputStream stream) {
+    public void storeObject(String objectId, InputStream stream) {
 
         File tempFile = File.createTempFile("test", ".tmp");
 

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/utils/PdfTextExtraction.java

@@ -1,35 +0,0 @@
-package com.iqser.red.service.ocr.v1.server.utils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.pdftron.common.PDFNetException;
-import com.pdftron.pdf.PDFDoc;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.PageIterator;
-import com.pdftron.pdf.TextExtractor;
-
-
-public class PdfTextExtraction {
-
-    public static String extractAllTextFromDocument(InputStream fileStream) throws IOException, PDFNetException {
-
-        PDFDoc pdfDoc = new PDFDoc(fileStream);
-        TextExtractor extractor = new TextExtractor();
-        List<String> texts = new ArrayList<>();
-
-        PageIterator iterator = pdfDoc.getPageIterator();
-        while (iterator.hasNext()) {
-            Page page = iterator.next();
-            extractor.begin(page);
-            texts.add(extractor.getAsText());
-        }
-
-        extractor.destroy();
-        pdfDoc.close();
-        return String.join("\n", texts);
-    }
-
-}

ocr-service-v1/ocr-service-server-v1/src/test/java/com/iqser/red/service/ocr/v1/server/utils/PdfUtils.java

@@ -1,18 +1,32 @@
 package com.iqser.red.service.ocr.v1.server.utils;
 
-import com.pdftron.pdf.ColorPt;
-import com.pdftron.pdf.ColorSpace;
-import com.pdftron.pdf.Element;
-import com.pdftron.pdf.ElementBuilder;
-import com.pdftron.pdf.ElementWriter;
-import com.pdftron.pdf.Page;
-import com.pdftron.pdf.Rect;
-import com.pdftron.pdf.RectCollection;
-
+import com.pdftron.common.PDFNetException;
+import com.pdftron.pdf.*;
 import lombok.SneakyThrows;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
 
-public class PdfDraw {
+public class PdfUtils {
+    public static String extractAllTextFromDocument(InputStream fileStream) throws IOException, PDFNetException {
+
+        PDFDoc pdfDoc = new PDFDoc(fileStream);
+        TextExtractor extractor = new TextExtractor();
+        List<String> texts = new ArrayList<>();
+
+        PageIterator iterator = pdfDoc.getPageIterator();
+        while (iterator.hasNext()) {
+            Page page = iterator.next();
+            extractor.begin(page);
+            texts.add(extractor.getAsText());
+        }
+
+        extractor.destroy();
+        pdfDoc.close();
+        return String.join("\n", texts);
+    }
 
     @SneakyThrows
     public static void drawGrid(ElementWriter writer, Page page) {
@@ -45,7 +59,6 @@ public class PdfDraw {
         eb.destroy();
     }
 
-
     @SneakyThrows
     public static void drawRectCollection(ElementWriter writer, RectCollection rectCollection) {
 
@@ -70,5 +83,4 @@ public class PdfDraw {
         colorPt.destroy();
         eb.destroy();
     }
-
 }

ocr-service-v1/pom.xml

@@ -7,7 +7,7 @@
     <parent>
         <groupId>com.iqser.red</groupId>
         <artifactId>platform-dependency</artifactId>
-        <version>1.17.0</version>
+        <version>1.16.0</version>
         <relativePath/>
     </parent>
 
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.22.0</version>
+                <version>1.20.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.iqser.red.service</groupId>
     <artifactId>ocr-service</artifactId>
-    <version>2.0-SNAPSHOT</version>
+    <version>1.0-SNAPSHOT</version>
 
     <packaging>pom</packaging>