Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-6864 - Optimization of Upload and Download for Large Files in Azure Blob Storage and AWS S3/MinIO

Browse Source 
- update to the new version for storage-commons
- replace getObject from storageService with downloadTo
- update tests
This commit is contained in:
Corina Olariu 2023-08-08 18:59:00 +03:00
parent e454533758
commit 2bcee8ccc2
15 changed files with 60 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java vendored
View File

@ -25,6 +25,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.iqser.red.persistence.service.v1.external.api.impl.service.OneTimeTokenService;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
@ -71,6 +72,7 @@ public class DownloadController implements DownloadResource {
private final AuditPersistenceService auditPersistenceService;
private final OneTimeTokenService oneTimeTokenDownloadService;
private final AccessControlService accessControlService;
private final FileManagementStorageService fileManagementStorageService;
@Value("${storage.backend:s3}")
private String storageBackend;
@ -225,7 +227,7 @@ public class DownloadController implements DownloadResource {
private InputStreamResource getFileForDownload(String storageId, String userId) {
try {
var response = storageService.getObject(TenantContext.getTenantId(), storageId);
var response = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId);
auditPersistenceService.audit(AuditRequest.builder()
.userId(userId)
@ -235,7 +237,7 @@ public class DownloadController implements DownloadResource {
.build());
downloadService.setDownloaded(JSONPrimitive.of(storageId));
return response;
return new InputStreamResource(response);
} catch (Exception e) {
throw new NotFoundException(e.getMessage(), e);
}

9
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java vendored
View File

@ -6,6 +6,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
import static com.iqser.red.service.persistence.management.v1.processor.service.FeignExceptionHandler.processFeignException;
import static com.iqser.red.service.persistence.management.v1.processor.utils.DownloadBufferUtils.fileProxyStreamForDownload;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -27,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.client.pdftronr
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
@ -65,6 +67,7 @@ public class FileManagementController implements FileManagementResource {
private final StorageService storageService;
private final FileStatusManagementService fileStatusManagementService;
private final FileStatusService fileStatusService;
private final FileManagementStorageService fileManagementStorageService;
@Timed
@ -119,7 +122,7 @@ public class FileManagementController implements FileManagementResource {
try {
var file = fileStatusManagementService.getFileStatus(fileId);
var untouchedFileStream = storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN));
var untouchedFileStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN));
return getResponseEntity(inline, untouchedFileStream, file.getFilename(), MediaType.APPLICATION_PDF);
} catch (FeignException e) {
if (e.status() == HttpStatus.NOT_FOUND.value()) {
@ -132,7 +135,7 @@ public class FileManagementController implements FileManagementResource {
@SneakyThrows
private ResponseEntity<?> getResponseEntity(boolean inline, InputStreamResource resource, String filename, MediaType mediaType) {
private ResponseEntity<?> getResponseEntity(boolean inline, FileInputStream resource, String filename, MediaType mediaType) {
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.setContentType(mediaType);
@ -141,7 +144,7 @@ public class FileManagementController implements FileManagementResource {
httpHeaders.add(DOWNLOAD_HEADER_NAME, inline ? "inline" : "attachment" + "; filename*=utf-8''" + StringEncodingUtils.urlEncode(filename));
}
return new ResponseEntity<>(fileProxyStreamForDownload(resource.getInputStream()), httpHeaders, HttpStatus.OK);
return new ResponseEntity<>(fileProxyStreamForDownload(resource), httpHeaders, HttpStatus.OK);
}

4
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/HighlightsController.java vendored
View File

@ -17,6 +17,7 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.Highlight
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
import com.iqser.red.service.persistence.service.v1.api.external.resource.HighlightsResource;
@ -39,6 +40,7 @@ public class HighlightsController implements HighlightsResource {
private final AccessControlService accessControlService;
private final FileStatusService fileStatusService;
private final ReanalysisService reanalysisService;
private final FileManagementStorageService fileManagementStorageService;
@SneakyThrows
@ -48,7 +50,7 @@ public class HighlightsController implements HighlightsResource {
fileStatusService.getStatus(fileId);
if (storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS))) {
return objectMapper.readValue(storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS)).getInputStream(),
return objectMapper.readValue(fileManagementStorageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS)),
Highlights.class);
}

6
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java vendored
View File

@ -19,6 +19,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
@ -40,8 +41,8 @@ import lombok.SneakyThrows;
public class RedactionLogController implements RedactionLogResource {
private final RedactionLogService redactionLogService;
private final StorageService storageService;
private final FileStatusService fileStatusService;
private final FileManagementStorageService fileManagementStorageService;
@PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')")
@ -199,9 +200,8 @@ public class RedactionLogController implements RedactionLogResource {
try {
String objectId = dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
var inputStreamResource = storageService.getObject(TenantContext.getTenantId(), objectId);
try (var inputStream = inputStreamResource.getInputStream()) {
try (var inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), objectId)) {
return zipBytes(filename, inputStream.readAllBytes());
}

5
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReportTemplateController.java vendored
View File

@ -27,6 +27,7 @@ import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionreportservice.PlaceholderClient;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionreportservice.ReportTemplatePlaceholderClient;
@ -68,6 +69,8 @@ public class ReportTemplateController implements ReportTemplateResource {
private final StorageService storageService;
private final ReportTemplatePersistenceService reportTemplatePersistenceService;
private final ReportTemplateService reportTemplateService;
private final FileManagementStorageService fileManagementStorageService;
@Override
@ -132,7 +135,7 @@ public class ReportTemplateController implements ReportTemplateResource {
try {
var reportTemplate = reportTemplatePersistenceService.find(templateId);
byte[] file = IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), reportTemplate.getStorageId()).getInputStream());
byte[] file = IOUtils.toByteArray(fileManagementStorageService.getObject(TenantContext.getTenantId(), reportTemplate.getStorageId()));
return getResponseEntity(file, reportTemplate.getFileName(), MediaType.APPLICATION_OCTET_STREAM);
} catch (StorageObjectDoesNotExist | IOException e) {
throw new NotFoundException("Template does not exist");

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ReduceTextFileSizeMigration10.java
View File

@ -1,11 +1,14 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import java.io.FileInputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.migration.migrations.model.reducetext.Text;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
@ -35,6 +38,9 @@ public class ReduceTextFileSizeMigration10 extends Migration {
@Autowired
private StorageService storageService;
@Autowired
private FileManagementStorageService fileManagementStorageService;
public ReduceTextFileSizeMigration10() {
@ -89,10 +95,10 @@ public class ReduceTextFileSizeMigration10 extends Migration {
try {
if (!storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT) + ".bak")) {
InputStreamResource textInputStreamResource = storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
FileInputStream textInputStreamResource = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
storageService.storeObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT) + ".bak",
textInputStreamResource.getInputStream());
textInputStreamResource);
var text = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), text);
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java
View File

@ -1,6 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
@ -19,7 +18,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
@ -61,6 +59,7 @@ public class DossierTemplateCloneService {
private final StorageService storageService;
private final DossierStatusPersistenceService dossierStatusPersistenceService;
private final WatermarkService watermarkService;
private final FileManagementStorageService fileManagementStorageService;
@Transactional
@ -224,14 +223,10 @@ public class DossierTemplateCloneService {
var reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplate.getId());
for (ReportTemplateEntity rte : reportTemplates) {
if (storageService.objectExists(TenantContext.getTenantId(), rte.getStorageId())) {
var storedReportTemplate = storageService.getObject(TenantContext.getTenantId(), rte.getStorageId());
var storedReportTemplate = fileManagementStorageService.getObject(TenantContext.getTenantId(), rte.getStorageId());
String storageId = clonedDossierTemplateId + "/" + rte.getFileName();
String templateId = UUID.randomUUID().toString();
try {
storageService.storeObject(TenantContext.getTenantId(), storageId, storedReportTemplate.getInputStream());
} catch (IOException e) {
throw new ConflictException("Could not clone report templates of dossier template.");
}
storageService.storeObject(TenantContext.getTenantId(), storageId, storedReportTemplate);
reportTemplatePersistenceService.insert(clonedDossierTemplateId, templateId, storageId, rte.getFileName(), rte.isActiveByDefault(), rte.isMultiFileReport());
} else {
log.debug("Deleted Report-template {} will not be cloned", rte.getTemplateId());

14
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import org.apache.commons.io.IOUtils;
@ -30,14 +32,22 @@ public class FileManagementStorageService {
@SneakyThrows
public byte[] getStoredObjectBytes(String dossierId, String fileId, FileType fileType) {
return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType)).getInputStream());
return IOUtils.toByteArray(getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType)));
}
@SneakyThrows
public byte[] getStoredObjectBytes(String storageId) {
return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream());
return IOUtils.toByteArray(getObject(TenantContext.getTenantId(), storageId));
}
@SneakyThrows
public FileInputStream getObject(String tenantId, String storageId) {
File destFile = File.createTempFile("destFile", ".data");
destFile.deleteOnExit();
storageService.downloadTo(tenantId, storageId, destFile);
return new FileInputStream(destFile);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -216,7 +216,7 @@ public class DownloadPreparationService {
var redactionResultDetail = redactionResultDetails.stream().filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType).findFirst();
if (redactionResultDetail.isPresent()) {
try (var inputStream = storageService.getObject(TenantContext.getTenantId(), redactionResultDetail.get().getStorageId()).getInputStream()) {
try (var inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), redactionResultDetail.get().getStorageId())) {
return inputStream.readAllBytes();
} catch (IOException e) {
throw new RuntimeException(e);

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java
View File

@ -57,7 +57,7 @@ public class LayoutParsingFinishedMessageReceiver {
log.info("Failed to process layout parsing request, errorCause: {}, timestamp: {}", errorCause, timestamp);
fileStatusProcessingUpdateService.analysisFailed(layoutParsingRequestIdentifierService.parseDossierId(analyzeRequest.identifier()),
layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()),
new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "redaction-service", timestamp));
new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "layoutparser-service", timestamp));
}

8
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java
View File

@ -31,6 +31,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTem
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.export.ExportDownloadMessageReceiver;
import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierAttributesConfig;
@ -101,6 +102,9 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
@Autowired
private StorageService storageService;
@Autowired
private FileManagementStorageService fileManagementStorageService;
@Autowired
private LegalBasisClient legalBasisClient;
@ -523,9 +527,9 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
var updated = dossierTemplateClient.createOrUpdateDossierTemplate(cru);
assertThat(updated.getName()).isNotEqualTo(dossierTemplate.getName());
var storedObject = storageService.getObject(TenantContext.getTenantId(), status.getStorageId());
var storedObject = fileManagementStorageService.getObject(TenantContext.getTenantId(), status.getStorageId());
var importTemplate = new MockMultipartFile("import.zip", "import.zip", "application/zip", storedObject.getInputStream());
var importTemplate = new MockMultipartFile("import.zip", "import.zip", "application/zip", storedObject);
var dossierImported = dossierTemplateClient.importDossierTemplate(importTemplate, dossierTemplate.getId(), true);
// assertThat(dossierImported).isNotNull();

4
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReduceFileSizeMigrationTest.java
View File

@ -36,10 +36,10 @@ public class ReduceFileSizeMigrationTest extends AbstractPersistenceServerServic
reduceTextFileSizeMigration10.migrateFile("dossierId", "fileId");
// readJSONObject will not work here.
var text = storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId("dossierId", "fileId", FileType.TEXT));
var text = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId("dossierId", "fileId", FileType.TEXT));
try (FileOutputStream fileOutputStream = new FileOutputStream(FileUtils.getTemporaryDirectory() + "/MigratedText.json")) {
fileOutputStream.write(text.getInputStream().readAllBytes());
fileOutputStream.write(text.readAllBytes());
}
}

3
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -57,6 +57,7 @@ import com.iqser.red.service.persistence.management.v1.processor.client.tenantus
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ApplicationConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository;
@ -132,6 +133,8 @@ public abstract class AbstractPersistenceServerServiceTest {
@Autowired
protected StorageService storageService;
@Autowired
protected FileManagementStorageService fileManagementStorageService;
@Autowired
protected DossierTemplateRepository dossierTemplateRepository;
@Autowired
protected DossierRepository dossierRepository;

7
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/FileSystemBackedStorageService.java
View File

@ -57,7 +57,7 @@ public class FileSystemBackedStorageService implements StorageService {
}
@Override
@SneakyThrows
public void storeObject(String tenantId, String objectId, InputStream stream) {
@ -73,14 +73,13 @@ public class FileSystemBackedStorageService implements StorageService {
@SneakyThrows
@Override
public InputStreamResource getObject(String tenantId, String objectId) {
public void downloadTo(String tenantId, String objectId, File destinationFile) {
var res = dataMap.get(objectId);
if (res == null) {
throw new StorageObjectDoesNotExist(new RuntimeException());
}
return new InputStreamResource(new FileInputStream(res));
IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
}

2
persistence-service-v1/pom.xml
View File

@ -41,7 +41,7 @@
<dependency>
<groupId>com.iqser.red</groupId>
<artifactId>platform-commons-dependency</artifactId>
<version>2.6.0</version>
<version>2.7.0</version>
<scope>import</scope>
<type>pom</type>
</dependency>