Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #654: RED-6497 Porting to 4.0

Browse Source 
Merge in RED/persistence-service from RED-6497 to master

* commit '78758a2a418a816f4efe0f7d64647926377ad4f7':
  RED-6497: Moved test to the same package as the class being tested
  RED-6497: Removed alternate file size check because it offers less error reporting capabilities
  RED-6497: Fixed issue where the file-size of download was saved incorrectly.
  RED-6497: Changed code to use a different method of getting a file size to have better error reporting
  RED-6497: Removed explicit variable initialization because its marked as a checkstyle violation.
  RED-6497: Removed alternate file size check because it offers less error reporting capabilities
This commit is contained in:
Viktor Seifert 2023-03-31 12:35:33 +02:00
commit a8c0ad42d1
7 changed files with 184 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -166,10 +166,7 @@ public class DownloadPreparationService {
private void updateStatusToReady(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
if (!Objects.equals(downloadStatus.getStatus(), DownloadStatusValue.READY)) {
downloadStatus.setStatus(DownloadStatusValue.READY);
}
downloadStatusPersistenceService.updateStatus(downloadStatus, DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java
View File

@ -221,7 +221,7 @@ public class DossierTemplateExportService {
}
storeZipFile(downloadStatus.getStorageId(), fileSystemBackedArchiver);
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
downloadStatusPersistenceService.updateStatus(downloadStatus, DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
} catch (JsonProcessingException e) {
log.debug("fail ", e);

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java
View File

@ -73,9 +73,11 @@ public class DownloadStatusPersistenceService {
@Transactional
public void updateStatus(String storageId, DownloadStatusValue status, long fileSize) {
public void updateStatus(DownloadStatusEntity entity, DownloadStatusValue statusValue, long fileSize) {
downloadStatusRepository.updateStatus(storageId, status, fileSize);
entity.setStatus(statusValue);
entity.setFileSize(fileSize);
downloadStatusRepository.save(entity);
}

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DownloadStatusRepository.java
View File

@ -18,12 +18,7 @@ public interface DownloadStatusRepository extends JpaRepository<DownloadStatusEn
@Modifying
@Query("update DownloadStatusEntity ds set ds.status = :status where ds.storageId = :storageId")
void updateStatus(String storageId, DownloadStatusValue status);
@Modifying
@Query("update DownloadStatusEntity ds set ds.status = :status, ds.fileSize = :fileSize where ds.storageId = :storageId")
void updateStatus(String storageId, DownloadStatusValue status, long fileSize);
@Modifying
@Query("update DownloadStatusEntity ds set ds.lastDownload = :lastDownload where ds.storageId = :storageId")

79
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java
View File

@ -6,6 +6,8 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
@ -21,14 +23,31 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class FileSystemBackedArchiver implements AutoCloseable {
private final boolean rethrowExceptions;
private final Set<String> createdFolders = new HashSet<>();
private final File tempFile;
private final ZipOutputStream zipOutputStream;
private long tempFileLength;
@SneakyThrows
public FileSystemBackedArchiver() {
this(false);
}
/**
* Controls whether exceptions are re-thrown. Mostly meant for testing.
*
* @param rethrowExceptions If true exceptions caught when handling streams and files will be re-thrown.
*/
@SneakyThrows
FileSystemBackedArchiver(boolean rethrowExceptions) {
this.rethrowExceptions = rethrowExceptions;
tempFile = FileUtils.createTempFile("archive", ".zip");
zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile));
}
@ -47,6 +66,14 @@ public class FileSystemBackedArchiver implements AutoCloseable {
}
@SneakyThrows
public InputStream toInputStream() {
closeStreamAndStoreTempFileLength();
return new BufferedInputStream(new FileInputStream(tempFile));
}
@SneakyThrows
public void addEntry(ArchiveModel archiveModel) {
// begin writing a new ZIP entry, positions the stream to the start of the entry entity
@ -65,41 +92,53 @@ public class FileSystemBackedArchiver implements AutoCloseable {
}
@SneakyThrows
public InputStream toInputStream() {
try {
zipOutputStream.close();
} catch (IOException e) {
log.debug(e.getMessage());
}
return new BufferedInputStream(new FileInputStream(tempFile));
}
@Override
public void close() {
closeStreamAndStoreTempFileLength();
try {
boolean res = tempFile.delete();
if (!res) {
log.warn("Failed to delete temp file");
Files.delete(tempFile.toPath());
} catch (IOException e) {
log.warn("Failed to delete temp-file", e);
if (rethrowExceptions) {
throw new RuntimeException(e);
}
zipOutputStream.close();
} catch (Exception e) {
log.debug("Failed to close FileSystemBackedArchiver");
}
}
public long getContentLength() {
closeStreamAndStoreTempFileLength();
return tempFileLength;
}
private void closeStreamAndStoreTempFileLength() {
try {
zipOutputStream.close();
} catch (IOException e) {
log.debug(e.getMessage());
log.warn("Failed to close temp-file stream", e);
if (rethrowExceptions) {
throw new RuntimeException(e);
}
}
if (tempFile.exists()) {
try {
var basicFileAttributes = Files.readAttributes(tempFile.toPath(), BasicFileAttributes.class);
tempFileLength = basicFileAttributes.size();
} catch (IOException e) {
if (rethrowExceptions) {
throw new RuntimeException(e);
}
}
} else {
log.warn("The temp file {} was deleted before it was completely processed", tempFile);
}
return tempFile.length();
}

13
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java
View File

@ -36,6 +36,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.storage.commons.service.StorageService;
@ -111,10 +112,10 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
.redactionPreviewColor("#aaaaaa")
.build());
var statuses = downloadClient.getDownloadStatus();
assertThat(statuses.getDownloadStatus()).isNotEmpty();
List<DownloadStatus> downloadStatuses = downloadClient.getDownloadStatus().getDownloadStatus();
assertThat(downloadStatuses).hasSize(1);
DownloadStatus firstDownloadStatus = statuses.getDownloadStatus().iterator().next();
DownloadStatus firstDownloadStatus = downloadStatuses.get(0);
assertThat(firstDownloadStatus.getLastDownload()).isNull();
String downloadId = firstDownloadStatus.getStorageId();
@ -134,6 +135,12 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
.redactionResultDetails(Collections.emptyList())
.build());
List<DownloadStatus> finalDownloadStatuses = downloadClient.getDownloadStatus().getDownloadStatus();
assertThat(finalDownloadStatuses).hasSize(1);
DownloadStatus finalDownloadStatus = finalDownloadStatuses.get(0);
assertThat(finalDownloadStatus.getStatus()).isEqualTo(DownloadStatusValue.READY);
assertThat(finalDownloadStatus.getFileSize()).isGreaterThan(0);
clearTenantContext();
}

108
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiverTest.java Normal file
View File

@ -0,0 +1,108 @@
package com.iqser.red.service.persistence.management.v1.processor.utils;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.InputStream;
import java.io.ObjectOutputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.SplittableRandom;
import org.junit.jupiter.api.Test;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class FileSystemBackedArchiverTest {
private final static byte[] dummyFileContent = new byte[]{1, 2};
@Test
@SneakyThrows
public void testFileSystemBackedArchiver() {
try (var fileSystemBackedArchiver = new FileSystemBackedArchiver(true)) {
SplittableRandom sr = new SplittableRandom();
var data = sr.doubles().limit(1024 * 1024).toArray();
for (int i = 0; i < 10; i++) {
log.info("At entry: {}, using {}MB of memory", i, (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024 * 1024));
try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(data);
byte[] bytes = bos.toByteArray();
var entry = new FileSystemBackedArchiver.ArchiveModel("folder-" + i, "file-" + i, bytes);
fileSystemBackedArchiver.addEntry(entry);
}
}
File tempFile = File.createTempFile("test", ".zip");
var contentSize = fileSystemBackedArchiver.getContentLength();
try (InputStream inputStream = fileSystemBackedArchiver.toInputStream()) {
Files.copy(inputStream, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
log.info("File: {}", tempFile.getAbsolutePath());
}
assertThat(tempFile.length()).isEqualTo(contentSize);
log.info("Total File Size: {}MB", tempFile.length() / (1024 * 1024));
Files.delete(tempFile.toPath());
}
}
@Test
public void testContentLengthForTwoEntries() {
try (var fileSystemBackedArchiver = new FileSystemBackedArchiver(true)) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", "original", dummyFileContent));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", "preview", dummyFileContent));
assertThat(fileSystemBackedArchiver.getContentLength()).isGreaterThan(0);
}
}
@Test
@SneakyThrows
public void testContentLengthForTwoEntriesAndStream() {
try (var fileSystemBackedArchiver = new FileSystemBackedArchiver(true)) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", "original", dummyFileContent));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", "preview", dummyFileContent));
try (InputStream stream = fileSystemBackedArchiver.toInputStream()) {
// Dummy statement to just have the code do something with the stream
//noinspection ResultOfMethodCallIgnored
stream.getClass();
}
assertThat(fileSystemBackedArchiver.getContentLength()).isGreaterThan(0);
}
}
@Test
public void testContentLengthForTwoEntriesWithClosing() {
// deliberately do not use try-with-resources to see if the content-length is available after temp file deletion
var fileSystemBackedArchiver = new FileSystemBackedArchiver(true);
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", "original", dummyFileContent));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", "preview", dummyFileContent));
fileSystemBackedArchiver.close();
assertThat(fileSystemBackedArchiver.getContentLength()).isGreaterThan(0);
}
}