Merge branch 'RED-6864' into 'main'

RED-6864 - Switch to DELETE_ON_CLOSE

See merge request fforesight/layout-parser!10

.gitlab-ci.yml

@@ -1,4 +1,21 @@
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'
-    file: 'ci-templates/gradle_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -16,7 +16,7 @@ dependencies {
         exclude("org.springframework.boot", "spring-boot-starter-validation")
     }
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.iqser.red.commons:storage-commons:2.27.0")
 
     implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -5,6 +5,9 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
@@ -16,14 +19,15 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -37,10 +41,11 @@ public class LayoutParsingStorageService {
 
     public PDDocument getOriginFile(String storageId) throws IOException {
 
-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var originDocumentInputStream = getObject(storageId)) {
             File tempFile = createTempFile("document", ".pdf");
             try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
                 IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
             }
             return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
         }
@@ -49,19 +54,22 @@ public class LayoutParsingStorageService {
 
     public ImageServiceResponse getImagesFile(String storageId) throws IOException {
 
-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (InputStream inputStream = getObject(storageId)) {
 
-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
         }
     }
 
 
     public TableServiceResponse getTablesFile(String storageId) throws IOException {
 
-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+        try (var tableClassificationStream = getObject(storageId)) {
 
+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
         }
     }
 
@@ -117,4 +125,13 @@ public class LayoutParsingStorageService {
         storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
     }
 
+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
 
 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
@@ -87,7 +88,8 @@ public class PropertiesMapper {
 
     public static String toString(Rectangle2D rectangle2D) {
 
-        return String.format("%f%s%f%s%f%s%f",
+        return String.format(Locale.US,
+                "%f%s%f%s%f%s%f",
                 rectangle2D.getX(),
                 DocumentStructure.RECTANGLE_DELIMITER,
                 rectangle2D.getY(),

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -19,7 +19,7 @@ dependencies {
     implementation(project(":layoutparser-service-processor"))
     implementation(project(":layoutparser-service-internal-api"))
 
-    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.iqser.red.commons:storage-commons:2.27.0")
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
     implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
     implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/FileSystemBackedStorageService.java

@@ -13,7 +13,6 @@ import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -36,14 +35,14 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @SneakyThrows
     @Override
-    public InputStreamResource getObject(String tenantId, String objectId) {
+    public void downloadTo(String tenantId, String objectId, File destinationFile) {
 
         var res = dataMap.get(objectId);
         if (res == null) {
             throw new StorageObjectDoesNotExist(new RuntimeException());
         }
-        return new InputStreamResource(new FileInputStream(res));
 
+        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
     }