Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-6864 - Close streams when done in order to delete temp files #65

Merged
andrei.isvoran.ext merged 3 commits from RED-6864 into master 2023-08-11 11:02:42 +02:00
Conversation 3 Commits 3 Files Changed 10 +60 -50
10 changed files with 60 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java vendored
View File

@ -4,6 +4,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_DOWNLOAD_STATUS;
import static com.iqser.red.service.persistence.management.v1.processor.utils.DownloadBufferUtils.fileProxyStreamForDownload;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -209,8 +210,9 @@ public class DownloadController implements DownloadResource {
TenantContext.setTenantId(tenantId);
var downloadStatus = getDownloadStatus(storageId, userId);
var fileDownloadStream = getFileForDownload(storageId, userId);
fileProxyStreamForDownload(fileDownloadStream);
return getResponseEntity(inline, fileDownloadStream, downloadStatus.getFilename(), MediaType.parseMediaType("application/zip"), downloadStatus.getFileSize());
return getResponseEntity(inline, new InputStreamResource(fileDownloadStream), downloadStatus.getFilename(), MediaType.parseMediaType("application/zip"), downloadStatus.getFileSize());
});
}
@ -224,7 +226,7 @@ public class DownloadController implements DownloadResource {
}
private InputStreamResource getFileForDownload(String storageId, String userId) {
private InputStream getFileForDownload(String storageId, String userId) {
try {
var response = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId);
@ -237,7 +239,7 @@ public class DownloadController implements DownloadResource {
.build());
downloadService.setDownloaded(JSONPrimitive.of(storageId));
return new InputStreamResource(response);
return response;
} catch (Exception e) {
throw new NotFoundException(e.getMessage(), e);
}
@ -279,10 +281,11 @@ public class DownloadController implements DownloadResource {
var token = oneTimeTokenDownloadService.getToken(oneTimeToken);
var downloadStatus = getDownloadStatus(token.getStorageId(), token.getUserId());
var fileDownloadStream = getFileForDownload(token.getStorageId(), token.getUserId());
fileProxyStreamForDownload(fileDownloadStream);
TenantContext.clear();
return getResponseEntity(inline, fileDownloadStream, downloadStatus.getFilename(), MediaType.parseMediaType("application/zip"), downloadStatus.getFileSize());
return getResponseEntity(inline, new InputStreamResource((fileDownloadStream)), downloadStatus.getFilename(), MediaType.parseMediaType("application/zip"), downloadStatus.getFileSize());
});
}

9
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java vendored
View File

@ -6,13 +6,12 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
import static com.iqser.red.service.persistence.management.v1.processor.service.FeignExceptionHandler.processFeignException;
import static com.iqser.red.service.persistence.management.v1.processor.utils.DownloadBufferUtils.fileProxyStreamForDownload;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
@ -123,7 +122,7 @@ public class FileManagementController implements FileManagementResource {
var file = fileStatusManagementService.getFileStatus(fileId);
var untouchedFileStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN));
return getResponseEntity(inline, untouchedFileStream, file.getFilename(), MediaType.APPLICATION_PDF);
return getResponseEntity(inline, untouchedFileStream, file.getFilename());
} catch (FeignException e) {
if (e.status() == HttpStatus.NOT_FOUND.value()) {
return new ResponseEntity<>(e.getMessage(), HttpStatus.NOT_FOUND);
@ -135,10 +134,10 @@ public class FileManagementController implements FileManagementResource {
@SneakyThrows
private ResponseEntity<?> getResponseEntity(boolean inline, FileInputStream resource, String filename, MediaType mediaType) {
private ResponseEntity<?> getResponseEntity(boolean inline, InputStream resource, String filename) {
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.setContentType(mediaType);
httpHeaders.setContentType(MediaType.APPLICATION_PDF);
if (filename != null) {
httpHeaders.add(DOWNLOAD_HEADER_NAME, inline ? "inline" : "attachment" + "; filename*=utf-8''" + StringEncodingUtils.urlEncode(filename));

8
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/HighlightsController.java vendored
View File

@ -7,6 +7,8 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
import static com.iqser.red.service.persistence.management.v1.processor.service.FeignExceptionHandler.processFeignException;
import static com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils.getStorageId;
import java.io.InputStream;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
@ -50,8 +52,10 @@ public class HighlightsController implements HighlightsResource {
fileStatusService.getStatus(fileId);
if (storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS))) {
return objectMapper.readValue(fileManagementStorageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS)),
Highlights.class);
InputStream stream = fileManagementStorageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.TEXT_HIGHLIGHTS));
Highlights highlights = objectMapper.readValue(stream, Highlights.class);
stream.close();
return highlights;
}
return new Highlights();

5
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java vendored
View File

@ -29,7 +29,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import feign.FeignException;
@ -202,7 +201,9 @@ public class RedactionLogController implements RedactionLogResource {
String objectId = dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
try (var inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), objectId)) {
return zipBytes(filename, inputStream.readAllBytes());
byte[] input = inputStream.readAllBytes();
inputStream.close();
return zipBytes(filename, input);
}
} catch (StorageObjectDoesNotExist e) {

10
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReportTemplateController.java vendored
View File

@ -135,18 +135,20 @@ public class ReportTemplateController implements ReportTemplateResource {
try {
var reportTemplate = reportTemplatePersistenceService.find(templateId);
byte[] file = IOUtils.toByteArray(fileManagementStorageService.getObject(TenantContext.getTenantId(), reportTemplate.getStorageId()));
return getResponseEntity(file, reportTemplate.getFileName(), MediaType.APPLICATION_OCTET_STREAM);
InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), reportTemplate.getStorageId());
byte[] file = IOUtils.toByteArray(inputStream);
inputStream.close();
return getResponseEntity(file, reportTemplate.getFileName());
} catch (StorageObjectDoesNotExist | IOException e) {
throw new NotFoundException("Template does not exist");
}
}
private ResponseEntity<?> getResponseEntity(byte[] file, String filename, MediaType mediaType) {
private ResponseEntity<?> getResponseEntity(byte[] file, String filename) {
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.setContentType(mediaType);
httpHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
if (filename != null) {
httpHeaders.add("Content-Disposition", "attachment; filename*=utf-8''" + StringEncodingUtils.urlEncode(filename));

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ReduceTextFileSizeMigration10.java
View File

@ -1,9 +1,8 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import java.io.FileInputStream;
import java.io.InputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
@ -95,10 +94,11 @@ public class ReduceTextFileSizeMigration10 extends Migration {
try {
if (!storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT) + ".bak")) {
FileInputStream textInputStreamResource = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
InputStream textInputStreamResource = fileManagementStorageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
storageService.storeObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT) + ".bak",
textInputStreamResource);
textInputStreamResource.close();
var text = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), text);
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java
View File

@ -39,6 +39,7 @@ import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ -217,7 +218,7 @@ public class DossierTemplateCloneService {
fileAttributeConfigPersistenceService.setFileAttributesConfig(clonedDossierTemplateId, facList);
}
@SneakyThrows
private void cloneReportTemplates(DossierTemplateEntity dossierTemplate, String clonedDossierTemplateId) {
var reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplate.getId());
@ -227,6 +228,7 @@ public class DossierTemplateCloneService {
String storageId = clonedDossierTemplateId + "/" + rte.getFileName();
String templateId = UUID.randomUUID().toString();
storageService.storeObject(TenantContext.getTenantId(), storageId, storedReportTemplate);
storedReportTemplate.close();
reportTemplatePersistenceService.insert(clonedDossierTemplateId, templateId, storageId, rte.getFileName(), rte.isActiveByDefault(), rte.isMultiFileReport());
} else {
log.debug("Deleted Report-template {} will not be cloned", rte.getTemplateId());

24
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -1,8 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Service;
@ -32,22 +34,28 @@ public class FileManagementStorageService {
@SneakyThrows
public byte[] getStoredObjectBytes(String dossierId, String fileId, FileType fileType) {
return IOUtils.toByteArray(getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType)));
InputStream inputStream = getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType));
byte[] storedObjectBytes = IOUtils.toByteArray(inputStream);
inputStream.close();
return storedObjectBytes;
}
@SneakyThrows
public byte[] getStoredObjectBytes(String storageId) {
return IOUtils.toByteArray(getObject(TenantContext.getTenantId(), storageId));
InputStream inputStream = getObject(TenantContext.getTenantId(), storageId);
byte[] storedObjectBytes = IOUtils.toByteArray(inputStream);
inputStream.close();
return storedObjectBytes;
}
@SneakyThrows
public FileInputStream getObject(String tenantId, String storageId) {
File destFile = File.createTempFile("destFile", ".data");
destFile.deleteOnExit();
storageService.downloadTo(tenantId, storageId, destFile);
return new FileInputStream(destFile);
public InputStream getObject(String tenantId, String storageId) {
File tempFile = File.createTempFile("temp", ".data");
storageService.downloadTo(tenantId, storageId, tempFile);
return Files.newInputStream(Paths.get(tempFile.getPath()), StandardOpenOption.DELETE_ON_CLOSE);
}

32
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -8,12 +8,9 @@ import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import jakarta.transaction.Transactional;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
@ -28,7 +25,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.ColorsS
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
@ -40,9 +36,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import jakarta.transaction.Transactional;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@ -56,13 +52,11 @@ import lombok.extern.slf4j.Slf4j;
public class DownloadPreparationService {
DownloadStatusPersistenceService downloadStatusPersistenceService;
FileStatusPersistenceService fileStatusPersistenceService;
FileManagementStorageService fileManagementStorageService;
ReportTemplatePersistenceService reportTemplatePersistenceService;
NotificationPersistenceService notificationPersistenceService;
RabbitTemplate rabbitTemplate;
ObjectMapper objectMapper;
StorageService storageService;
DownloadReportCleanupService downloadReportCleanupService;
ColorsService colorsService;
FileManagementServiceSettings settings;
@ -70,7 +64,7 @@ public class DownloadPreparationService {
@Transactional
public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException {
public void createDownload(ReportResultMessage reportResultMessage) {
var downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
var dossier = downloadStatus.getDossier();
@ -113,18 +107,12 @@ public class DownloadPreparationService {
var result = new LinkedList<RedactionType>();
for (var downloadFileType : downloadFileTypes) {
switch (downloadFileType) {
case REDACTED:
result.add(RedactionType.REDACTED);
break;
case PREVIEW:
result.add(RedactionType.PREVIEW);
break;
case DELTA_PREVIEW:
result.add(RedactionType.DELTA);
break;
default:
// Other types don't need to be passed to pdf-tron
break;
case REDACTED -> result.add(RedactionType.REDACTED);
case PREVIEW -> result.add(RedactionType.PREVIEW);
case DELTA_PREVIEW -> result.add(RedactionType.DELTA);
default -> {
}
// Other types don't need to be passed to pdf-tron
}
}
@ -217,7 +205,9 @@ public class DownloadPreparationService {
var redactionResultDetail = redactionResultDetails.stream().filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType).findFirst();
if (redactionResultDetail.isPresent()) {
try (var inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), redactionResultDetail.get().getStorageId())) {
return inputStream.readAllBytes();
byte[] storedFileBytes = inputStream.readAllBytes();
inputStream.close();
return storedFileBytes;
} catch (IOException e) {
throw new RuntimeException(e);
}

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/DownloadBufferUtils.java
View File

@ -19,6 +19,7 @@ public final class DownloadBufferUtils {
var tempFile = FileUtils.createTempFile("download", "pdf");
org.apache.commons.io.FileUtils.copyInputStreamToFile(inputStream, tempFile);
inputStream.close();
if (RequestContextHolder.getRequestAttributes() != null) {
RequestContextHolder.getRequestAttributes().setAttribute(DOWNLOAD_TEMP_FILE_BUFFER_LOCATION, tempFile.getAbsolutePath(), 0);