From cfca5376a0c9887d46714195285ec2e01c7aabec Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Tue, 8 Aug 2023 17:16:40 +0200 Subject: [PATCH] RED-6864 - Switch to new storage-commons download --- .../build.gradle.kts | 2 +- .../processor/LayoutParsingStorageService.java | 17 ++++++++++++++--- .../build.gradle.kts | 2 +- .../utils/FileSystemBackedStorageService.java | 5 ++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/layoutparser-service/layoutparser-service-processor/build.gradle.kts b/layoutparser-service/layoutparser-service-processor/build.gradle.kts index 8d0692a..9baa04a 100644 --- a/layoutparser-service/layoutparser-service-processor/build.gradle.kts +++ b/layoutparser-service/layoutparser-service-processor/build.gradle.kts @@ -16,7 +16,7 @@ dependencies { exclude("org.springframework.boot", "spring-boot-starter-validation") } implementation("com.knecon.fforesight:tenant-commons:0.10.0") - implementation("com.iqser.red.commons:storage-commons:2.1.0") + implementation("com.iqser.red.commons:storage-commons:2.27.0") implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}") implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}") diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java index f2517ee..7053c01 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java @@ -1,6 +1,7 @@ package com.knecon.fforesight.service.layoutparser.processor; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -24,6 +25,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -37,7 +39,7 @@ public class LayoutParsingStorageService { public PDDocument getOriginFile(String storageId) throws IOException { - try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) { + try (var originDocumentInputStream = getObject(storageId)) { File tempFile = createTempFile("document", ".pdf"); try (var tempFileOutputStream = new FileOutputStream(tempFile)) { IOUtils.copy(originDocumentInputStream, tempFileOutputStream); @@ -49,7 +51,7 @@ public class LayoutParsingStorageService { public ImageServiceResponse getImagesFile(String storageId) throws IOException { - try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) { + try (InputStream inputStream = getObject(storageId)) { return objectMapper.readValue(inputStream, ImageServiceResponse.class); } @@ -58,7 +60,7 @@ public class LayoutParsingStorageService { public TableServiceResponse getTablesFile(String storageId) throws IOException { - try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) { + try (var tableClassificationStream = getObject(storageId)) { return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class); @@ -117,4 +119,13 @@ public class LayoutParsingStorageService { storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText); } + @SneakyThrows + private InputStream getObject(String storageId) { + + File tempFile = File.createTempFile("temp", ".data"); + tempFile.deleteOnExit(); + storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile); + return new FileInputStream(tempFile); + } + } diff --git a/layoutparser-service/layoutparser-service-server/build.gradle.kts b/layoutparser-service/layoutparser-service-server/build.gradle.kts index 250c6fe..1ba1c4a 100644 --- a/layoutparser-service/layoutparser-service-server/build.gradle.kts +++ b/layoutparser-service/layoutparser-service-server/build.gradle.kts @@ -19,7 +19,7 @@ dependencies { implementation(project(":layoutparser-service-processor")) implementation(project(":layoutparser-service-internal-api")) - implementation("com.iqser.red.commons:storage-commons:2.1.0") + implementation("com.iqser.red.commons:storage-commons:2.27.0") implementation("com.knecon.fforesight:tenant-commons:0.10.0") implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2") implementation("com.amazonaws:aws-java-sdk-s3:1.12.514") diff --git a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/FileSystemBackedStorageService.java b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/FileSystemBackedStorageService.java index 9f12673..674dbea 100644 --- a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/FileSystemBackedStorageService.java +++ b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/FileSystemBackedStorageService.java @@ -13,7 +13,6 @@ import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; -import org.springframework.core.io.InputStreamResource; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; @@ -36,14 +35,14 @@ public class FileSystemBackedStorageService implements StorageService { @SneakyThrows @Override - public InputStreamResource getObject(String tenantId, String objectId) { + public void downloadTo(String tenantId, String objectId, File destinationFile) { var res = dataMap.get(objectId); if (res == null) { throw new StorageObjectDoesNotExist(new RuntimeException()); } - return new InputStreamResource(new FileInputStream(res)); + IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile)); }