From 2bcee8ccc2cb0182e147de00db2c57b9e5627e1f Mon Sep 17 00:00:00 2001 From: Corina Olariu Date: Tue, 8 Aug 2023 18:59:00 +0300 Subject: [PATCH] RED-6864 - Optimization of Upload and Download for Large Files in Azure Blob Storage and AWS S3/MinIO - update to the new version for storage-commons - replace getObject from storageService with downloadTo - update tests --- .../api/impl/controller/DownloadController.java | 6 ++++-- .../impl/controller/FileManagementController.java | 9 ++++++--- .../api/impl/controller/HighlightsController.java | 4 +++- .../impl/controller/RedactionLogController.java | 6 +++--- .../impl/controller/ReportTemplateController.java | 5 ++++- .../migrations/ReduceTextFileSizeMigration10.java | 10 ++++++++-- .../service/DossierTemplateCloneService.java | 11 +++-------- .../service/FileManagementStorageService.java | 14 ++++++++++++-- .../download/DownloadPreparationService.java | 2 +- .../LayoutParsingFinishedMessageReceiver.java | 2 +- .../integration/tests/DossierTemplateTest.java | 8 ++++++-- .../tests/ReduceFileSizeMigrationTest.java | 4 ++-- .../AbstractPersistenceServerServiceTest.java | 3 +++ .../utils/FileSystemBackedStorageService.java | 7 +++---- persistence-service-v1/pom.xml | 2 +- 15 files changed, 60 insertions(+), 33 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java index 008a58fff..35bc4db6e 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java @@ -25,6 +25,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; import com.iqser.red.persistence.service.v1.external.api.impl.service.OneTimeTokenService; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; @@ -71,6 +72,7 @@ public class DownloadController implements DownloadResource { private final AuditPersistenceService auditPersistenceService; private final OneTimeTokenService oneTimeTokenDownloadService; private final AccessControlService accessControlService; + private final FileManagementStorageService fileManagementStorageService; @Value("${storage.backend:s3}") private String storageBackend; @@ -225,7 +227,7 @@ public class DownloadController implements DownloadResource { private InputStreamResource getFileForDownload(String storageId, String userId) { try { - var response = storageService.getObject(TenantContext.getTenantId(), storageId); + var response = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId); auditPersistenceService.audit(AuditRequest.builder() .userId(userId) @@ -235,7 +237,7 @@ public class DownloadController implements DownloadResource { .build()); downloadService.setDownloaded(JSONPrimitive.of(storageId)); - return response; + return new InputStreamResource(response); } catch (Exception e) { throw new NotFoundException(e.getMessage(), e); } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java index efe04be6e..0b06ec2ff 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java @@ -6,6 +6,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac import static com.iqser.red.service.persistence.management.v1.processor.service.FeignExceptionHandler.processFeignException; import static com.iqser.red.service.persistence.management.v1.processor.utils.DownloadBufferUtils.fileProxyStreamForDownload; +import java.io.FileInputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.client.pdftronr import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.FileService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; @@ -65,6 +67,7 @@ public class FileManagementController implements FileManagementResource { private final StorageService storageService; private final FileStatusManagementService fileStatusManagementService; private final FileStatusService fileStatusService; + private final FileManagementStorageService fileManagementStorageService; @Timed @@ -119,7 +122,7 @@ public class FileManagementController implements FileManagementResource { try { var file = fileStatusManagementService.getFileStatus(fileId); - var untouchedFileStream = storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN)); + var untouchedFileStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN)); return getResponseEntity(inline, untouchedFileStream, file.getFilename(), MediaType.APPLICATION_PDF); } catch (FeignException e) { if (e.status() == HttpStatus.NOT_FOUND.value()) { @@ -132,7 +135,7 @@ public class FileManagementController implements FileManagementResource { @SneakyThrows - private ResponseEntity getResponseEntity(boolean inline, InputStreamResource resource, String filename, MediaType mediaType) { + private ResponseEntity getResponseEntity(boolean inline, FileInputStream resource, String filename, MediaType mediaType) { HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.setContentType(mediaType); @@ -141,7 +144,7 @@ public class FileManagementController implements FileManagementResource { httpHeaders.add(DOWNLOAD_HEADER_NAME, inline ? "inline" : "attachment" + "; filename*=utf-8''" + StringEncodingUtils.urlEncode(filename)); } - return new ResponseEntity<>(fileProxyStreamForDownload(resource.getInputStream()), httpHeaders, HttpStatus.OK); + return new ResponseEntity<>(fileProxyStreamForDownload(resource), httpHeaders, HttpStatus.OK); } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/HighlightsController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/HighlightsController.java index 0344eb89e..1dfa8e6ea 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/HighlightsController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/HighlightsController.java @@ -17,6 +17,7 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.Highlight import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; import com.iqser.red.service.persistence.service.v1.api.external.resource.HighlightsResource; @@ -39,6 +40,7 @@ public class HighlightsController implements HighlightsResource { private final AccessControlService accessControlService; private final FileStatusService fileStatusService; private final ReanalysisService reanalysisService; + private final FileManagementStorageService fileManagementStorageService; @SneakyThrows @@ -48,7 +50,7 @@ public class HighlightsController implements HighlightsResource { fileStatusService.getStatus(fileId); if (storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS))) { - return objectMapper.readValue(storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS)).getInputStream(), + return objectMapper.readValue(fileManagementStorageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS)), Highlights.class); } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java index f5406748f..50e0266ea 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java @@ -19,6 +19,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService; import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils; @@ -40,8 +41,8 @@ import lombok.SneakyThrows; public class RedactionLogController implements RedactionLogResource { private final RedactionLogService redactionLogService; - private final StorageService storageService; private final FileStatusService fileStatusService; + private final FileManagementStorageService fileManagementStorageService; @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')") @@ -199,9 +200,8 @@ public class RedactionLogController implements RedactionLogResource { try { String objectId = dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension(); - var inputStreamResource = storageService.getObject(TenantContext.getTenantId(), objectId); - try (var inputStream = inputStreamResource.getInputStream()) { + try (var inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), objectId)) { return zipBytes(filename, inputStream.readAllBytes()); } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReportTemplateController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReportTemplateController.java index 445125f4e..6ba129984 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReportTemplateController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReportTemplateController.java @@ -27,6 +27,7 @@ import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; import com.iqser.red.service.persistence.management.v1.processor.client.redactionreportservice.PlaceholderClient; import com.iqser.red.service.persistence.management.v1.processor.client.redactionreportservice.ReportTemplatePlaceholderClient; @@ -68,6 +69,8 @@ public class ReportTemplateController implements ReportTemplateResource { private final StorageService storageService; private final ReportTemplatePersistenceService reportTemplatePersistenceService; private final ReportTemplateService reportTemplateService; + private final FileManagementStorageService fileManagementStorageService; + @Override @@ -132,7 +135,7 @@ public class ReportTemplateController implements ReportTemplateResource { try { var reportTemplate = reportTemplatePersistenceService.find(templateId); - byte[] file = IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), reportTemplate.getStorageId()).getInputStream()); + byte[] file = IOUtils.toByteArray(fileManagementStorageService.getObject(TenantContext.getTenantId(), reportTemplate.getStorageId())); return getResponseEntity(file, reportTemplate.getFileName(), MediaType.APPLICATION_OCTET_STREAM); } catch (StorageObjectDoesNotExist | IOException e) { throw new NotFoundException("Template does not exist"); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ReduceTextFileSizeMigration10.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ReduceTextFileSizeMigration10.java index c3865f96e..f19e2adf5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ReduceTextFileSizeMigration10.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ReduceTextFileSizeMigration10.java @@ -1,11 +1,14 @@ package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; +import java.io.FileInputStream; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.model.reducetext.Text; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; @@ -35,6 +38,9 @@ public class ReduceTextFileSizeMigration10 extends Migration { @Autowired private StorageService storageService; + @Autowired + private FileManagementStorageService fileManagementStorageService; + public ReduceTextFileSizeMigration10() { @@ -89,10 +95,10 @@ public class ReduceTextFileSizeMigration10 extends Migration { try { if (!storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT) + ".bak")) { - InputStreamResource textInputStreamResource = storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT)); + FileInputStream textInputStreamResource = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT)); storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT) + ".bak", - textInputStreamResource.getInputStream()); + textInputStreamResource); var text = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class); storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), text); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java index 697b0321a..f03b5a527 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java @@ -1,6 +1,5 @@ package com.iqser.red.service.persistence.management.v1.processor.service; -import java.io.IOException; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -19,7 +18,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService; @@ -61,6 +59,7 @@ public class DossierTemplateCloneService { private final StorageService storageService; private final DossierStatusPersistenceService dossierStatusPersistenceService; private final WatermarkService watermarkService; + private final FileManagementStorageService fileManagementStorageService; @Transactional @@ -224,14 +223,10 @@ public class DossierTemplateCloneService { var reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplate.getId()); for (ReportTemplateEntity rte : reportTemplates) { if (storageService.objectExists(TenantContext.getTenantId(), rte.getStorageId())) { - var storedReportTemplate = storageService.getObject(TenantContext.getTenantId(), rte.getStorageId()); + var storedReportTemplate = fileManagementStorageService.getObject(TenantContext.getTenantId(), rte.getStorageId()); String storageId = clonedDossierTemplateId + "/" + rte.getFileName(); String templateId = UUID.randomUUID().toString(); - try { - storageService.storeObject(TenantContext.getTenantId(), storageId, storedReportTemplate.getInputStream()); - } catch (IOException e) { - throw new ConflictException("Could not clone report templates of dossier template."); - } + storageService.storeObject(TenantContext.getTenantId(), storageId, storedReportTemplate); reportTemplatePersistenceService.insert(clonedDossierTemplateId, templateId, storageId, rte.getFileName(), rte.isActiveByDefault(), rte.isMultiFileReport()); } else { log.debug("Deleted Report-template {} will not be cloned", rte.getTemplateId()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java index 0571a3abf..26ad84a4d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import java.io.File; +import java.io.FileInputStream; import java.io.InputStream; import org.apache.commons.io.IOUtils; @@ -30,14 +32,22 @@ public class FileManagementStorageService { @SneakyThrows public byte[] getStoredObjectBytes(String dossierId, String fileId, FileType fileType) { - return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType)).getInputStream()); + return IOUtils.toByteArray(getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType))); } @SneakyThrows public byte[] getStoredObjectBytes(String storageId) { - return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()); + return IOUtils.toByteArray(getObject(TenantContext.getTenantId(), storageId)); + } + + @SneakyThrows + public FileInputStream getObject(String tenantId, String storageId) { + File destFile = File.createTempFile("destFile", ".data"); + destFile.deleteOnExit(); + storageService.downloadTo(tenantId, storageId, destFile); + return new FileInputStream(destFile); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java index fdab09f2a..7ddfb8a3b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java @@ -216,7 +216,7 @@ public class DownloadPreparationService { var redactionResultDetail = redactionResultDetails.stream().filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType).findFirst(); if (redactionResultDetail.isPresent()) { - try (var inputStream = storageService.getObject(TenantContext.getTenantId(), redactionResultDetail.get().getStorageId()).getInputStream()) { + try (var inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), redactionResultDetail.get().getStorageId())) { return inputStream.readAllBytes(); } catch (IOException e) { throw new RuntimeException(e); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index 7f4772ae3..be516e0e6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -57,7 +57,7 @@ public class LayoutParsingFinishedMessageReceiver { log.info("Failed to process layout parsing request, errorCause: {}, timestamp: {}", errorCause, timestamp); fileStatusProcessingUpdateService.analysisFailed(layoutParsingRequestIdentifierService.parseDossierId(analyzeRequest.identifier()), layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()), - new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "redaction-service", timestamp)); + new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "layoutparser-service", timestamp)); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java index 63a91882e..c0b008f31 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java @@ -31,6 +31,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTem import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.export.ExportDownloadMessageReceiver; import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierAttributesConfig; @@ -101,6 +102,9 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { @Autowired private StorageService storageService; + @Autowired + private FileManagementStorageService fileManagementStorageService; + @Autowired private LegalBasisClient legalBasisClient; @@ -523,9 +527,9 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest { var updated = dossierTemplateClient.createOrUpdateDossierTemplate(cru); assertThat(updated.getName()).isNotEqualTo(dossierTemplate.getName()); - var storedObject = storageService.getObject(TenantContext.getTenantId(), status.getStorageId()); + var storedObject = fileManagementStorageService.getObject(TenantContext.getTenantId(), status.getStorageId()); - var importTemplate = new MockMultipartFile("import.zip", "import.zip", "application/zip", storedObject.getInputStream()); + var importTemplate = new MockMultipartFile("import.zip", "import.zip", "application/zip", storedObject); var dossierImported = dossierTemplateClient.importDossierTemplate(importTemplate, dossierTemplate.getId(), true); // assertThat(dossierImported).isNotNull(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReduceFileSizeMigrationTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReduceFileSizeMigrationTest.java index ec4261f6a..c1241731f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReduceFileSizeMigrationTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReduceFileSizeMigrationTest.java @@ -36,10 +36,10 @@ public class ReduceFileSizeMigrationTest extends AbstractPersistenceServerServic reduceTextFileSizeMigration10.migrateFile("dossierId", "fileId"); // readJSONObject will not work here. - var text = storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId("dossierId", "fileId", FileType.TEXT)); + var text = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId("dossierId", "fileId", FileType.TEXT)); try (FileOutputStream fileOutputStream = new FileOutputStream(FileUtils.getTemporaryDirectory() + "/MigratedText.json")) { - fileOutputStream.write(text.getInputStream().readAllBytes()); + fileOutputStream.write(text.readAllBytes()); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index fc1516ba3..cf95526f0 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -57,6 +57,7 @@ import com.iqser.red.service.persistence.management.v1.processor.client.tenantus import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity; import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles; import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ApplicationConfigRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository; @@ -132,6 +133,8 @@ public abstract class AbstractPersistenceServerServiceTest { @Autowired protected StorageService storageService; @Autowired + protected FileManagementStorageService fileManagementStorageService; + @Autowired protected DossierTemplateRepository dossierTemplateRepository; @Autowired protected DossierRepository dossierRepository; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/FileSystemBackedStorageService.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/FileSystemBackedStorageService.java index 0867b5ea4..b5a7f7c66 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/FileSystemBackedStorageService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/FileSystemBackedStorageService.java @@ -57,7 +57,7 @@ public class FileSystemBackedStorageService implements StorageService { } - @Override + @SneakyThrows public void storeObject(String tenantId, String objectId, InputStream stream) { @@ -73,14 +73,13 @@ public class FileSystemBackedStorageService implements StorageService { @SneakyThrows @Override - public InputStreamResource getObject(String tenantId, String objectId) { + public void downloadTo(String tenantId, String objectId, File destinationFile) { var res = dataMap.get(objectId); if (res == null) { throw new StorageObjectDoesNotExist(new RuntimeException()); } - return new InputStreamResource(new FileInputStream(res)); - + IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile)); } diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index 0409f780a..bd22d60dc 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -41,7 +41,7 @@ com.iqser.red platform-commons-dependency - 2.6.0 + 2.7.0 import pom