Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-10127' into 'master'

Browse Source 
RED-10127: Import/Export of dossiertemplate with preview files

Closes RED-10127

See merge request redactmanager/persistence-service!782
This commit is contained in:
Kilian Schüttler 2024-10-14 15:30:18 +02:00
commit abda71fca3
16 changed files with 392 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java vendored
View File

@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
@ -48,6 +49,7 @@ public class SupportController implements SupportResource {
private final FileStatusManagementService fileStatusManagementService;
private final FileExchangeExportService fileExchangeExportService;
private final FileExchangeImportService fileExchangeImportService;
private final DatasetExchangeService datasetExchangeService;
@Override
@ -127,6 +129,13 @@ public class SupportController implements SupportResource {
}
@Override
public DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
return datasetExchangeService.prepareExport(dossierTemplateId);
}
@Override
public DownloadResponse exportFiles(String dossierTemplateId, FileExchangeExportRequest exportRequest) {
@ -147,4 +156,17 @@ public class SupportController implements SupportResource {
return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
}
@Override
@PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
public ImportResponse importDataset(MultipartFile file) {
byte[] bytes;
try {
bytes = file.getBytes();
} catch (IOException e) {
throw new BadRequestException("File could not be read and is likely corrupted.", e);
}
return datasetExchangeService.importDataset(KeycloakSecurity.getUserId(), bytes);
}
}

23
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java vendored
View File

@ -34,6 +34,7 @@ public interface SupportResource {
String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status/filter";
String FILE_EXCHANGE_REST_PATH = ExternalApi.BASE_PATH + "/file-exchange";
String DATASET_EXCHANGE = ExternalApi.BASE_PATH + "/dataset-exchange";
String BULK_REST_PATH = "/bulk";
@ -143,6 +144,21 @@ public interface SupportResource {
DownloadResponse exportFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody FileExchangeExportRequest exportRequest);
@ResponseStatus(value = HttpStatus.OK)
@ResponseBody
@PostMapping(value = DATASET_EXCHANGE
+ EXPORT
+ DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
## Export Preview Files Endpoint
Use this endpoint to export all files of a DossierTemplate as a Preview file containing all applied annotations as redaction annotations with additional data
The endpoint returns a String storageId, which is used to query the DownloadController for the export zip archive's status and to download the archive.
""")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@ -150,4 +166,11 @@ public interface SupportResource {
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. Returns the resulting dossierTemplateId.")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ImportResponse importDataset(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
}

57
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java Normal file
View File

@ -0,0 +1,57 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models;
import java.util.LinkedList;
import java.util.List;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DatasetExchangeImportModel {
private ImportTemplateResult importTemplateResult = ImportTemplateResult.builder().build();
private List<Dossier> dossiers = new LinkedList<>();
@Builder
@Getter
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public static class Dossier {
String name;
@Builder.Default
List<File> files = new LinkedList<>();
}
@Getter
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public static class File {
String name;
byte[] data;
}
public DatasetExchangeImportModel.Dossier getOrCreateDossier(String name) {
for (Dossier dossier : dossiers) {
if (dossier.getName().equals(name)) {
return dossier;
}
}
Dossier dossier = Dossier.builder().name(name).build();
dossiers.add(dossier);
return dossier;
}
}

183
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java Normal file
View File

@ -0,0 +1,183 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import static com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeArchivalService.SIZE_THRESHOLD;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DatasetExchangeArchiveReader;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadPreparationService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DatasetExchangeService {
DossierTemplateManagementService dossierTemplateManagementService;
DownloadStatusPersistenceService downloadStatusPersistenceService;
FileStatusPersistenceService fileStatusPersistenceService;
DossierPersistenceService dossierPersistenceService;
ColorsService colorsService;
RabbitTemplate rabbitTemplate;
FileManagementServiceSettings settings;
DossierTemplateImportService dossierTemplateImportService;
DossierCreatorService dossierCreatorService;
ReportTemplatePersistenceService reportTemplateService;
UploadService uploadService;
DownloadPreparationService downloadPreparationService;
FileManagementStorageService storageService;
public DownloadResponse prepareExport(String dossierTemplateId) {
var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId);
String downloadFilename = template.getName() + "_PREVIEW_FILES.zip";
String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES");
List<FileEntity> files = fileStatusPersistenceService.getStatusesForDossierTemplate(dossierTemplateId);
List<DossierEntity> dossiers = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
if (dossiers.isEmpty()) {
throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId);
}
ColorsEntity colors = colorsService.getColors(dossierTemplateId);
downloadPreparationService.clearRedactionStatusEntries(storageId);
if (storageService.objectExists(storageId)) {
storageService.deleteObject(storageId);
}
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.createStatus(KeycloakSecurity.getUserId(),
storageId,
dossiers.get(0),
downloadFilename,
"application/zip",
files.stream()
.map(FileEntity::getId)
.toList(),
Set.of(DownloadFileType.DATASET_EXPORT),
Collections.emptyList(),
colors.getPreviewColor());
RedactionMessage.RedactionMessageBuilder builder = RedactionMessage.builder()
.appliedRedactionColor(colors.getRedactionColor())
.redactionPreviewColor(colors.getPreviewColor())
.keepHiddenText(true)
.keepOverlappingObjects(true)
.keepImageMetaData(true)
.downloadId(storageId)
.redactionTypes(List.of(RedactionType.PREVIEW));
downloadStatus.getFiles()
.forEach(fileEntity -> {
RedactionMessage message = builder.dossierId(fileEntity.getDossierId())
.fileId(fileEntity.getId())
.unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED)
.build();
log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", storageId, fileEntity.getId());
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_REQUEST_EXCHANGE, TenantContext.getTenantId(), message);
});
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.GENERATING);
return new DownloadResponse(storageId);
}
@SneakyThrows
public ImportResponse importDataset(String userId, byte[] archive) {
DatasetExchangeArchiveReader datasetExchangeArchiveReader = new DatasetExchangeArchiveReader(userId);
try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
zipEntryIterator.forEachRemaining(datasetExchangeArchiveReader::handleZipEntryData);
}
DatasetExchangeImportModel importModel = datasetExchangeArchiveReader.getDatasetExchangeImportModel();
TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importModel.getImportTemplateResult());
for (DatasetExchangeImportModel.Dossier dossier : importModel.getDossiers()) {
String dossierId = getDossierForImport(templateImportInfo, dossier, userId);
for (DatasetExchangeImportModel.File file : dossier.getFiles()) {
uploadService.processSingleFile(dossierId, file.getName(), file.getData(), false, false);
}
}
return new ImportResponse(templateImportInfo.getDossierTemplateId());
}
private String getDossierForImport(TemplateImportInfo templateImportInfo, DatasetExchangeImportModel.Dossier dossier, String userId) {
String dossierTemplateId = templateImportInfo.getDossierTemplateId();
CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
.dossierName(dossier.getName())
.description(dossier.getName())
.downloadFileTypes(Collections.emptySet())
.dossierTemplateId(dossierTemplateId)
.requestingUser(userId)
.reportTemplateIds(reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
.stream()
.map(ReportTemplateEntity::getTemplateId)
.toList())
.build();
Dossier importedDossier = null;
int retries = 0;
while (importedDossier == null && retries < 100) {
try {
importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
} catch (ConflictException e) {
retries++;
request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
}
}
if (importedDossier == null) {
throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
}
return importedDossier.getId();
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java
View File

@ -67,7 +67,7 @@ public class DossierImportService {
Dossier importedDossier = null;
int retries = 0;
while (importedDossier == null && retries < 10) {
while (importedDossier == null && retries < 100) {
try {
importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
} catch (ConflictException e) {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java
View File

@ -72,6 +72,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@ -150,8 +151,9 @@ public class DossierTemplateExportService {
}
@SneakyThrows
@Observed(name = "DossierTemplateExportService", contextualName = "write-dossier-template-to-archive")
public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) throws IOException {
public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) {
// add dossier template name and meta data
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,

50
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java Normal file
View File

@ -0,0 +1,50 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel;
import lombok.AccessLevel;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DatasetExchangeArchiveReader {
DossierTemplateArchiveReader dossierTemplateArchiveReader;
DatasetExchangeImportModel datasetExchangeImportModel;
public DatasetExchangeArchiveReader(String userId) {
dossierTemplateArchiveReader = new DossierTemplateArchiveReader(userId, null, false);
datasetExchangeImportModel = new DatasetExchangeImportModel();
}
public DatasetExchangeImportModel getDatasetExchangeImportModel() {
datasetExchangeImportModel.setImportTemplateResult(dossierTemplateArchiveReader.buildResult());
return datasetExchangeImportModel;
}
@SneakyThrows
public void handleZipEntryData(ZipEntryData zipEntryData) {
if (zipEntryData.getBaseFolder().equals(FileExchangeNames.TEMPLATE_FOLDER)) {
dossierTemplateArchiveReader.handleZipEntryData(zipEntryData);
} else if (zipEntryData.getDepth() >= 1) { // in dossier folders
String dossierName = zipEntryData.getBaseFolder().replaceAll("\\s*\\([1-9]+\\)", "");
String fileName = zipEntryData.getFileName();
if (!fileName.endsWith(".pdf")) {
return;
}
var dossier = datasetExchangeImportModel.getOrCreateDossier(dossierName);
dossier.getFiles().add(new DatasetExchangeImportModel.File(fileName, zipEntryData.data()));
}
}
}

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java
View File

@ -1,6 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders;
import java.io.*;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.UUID;
@ -70,7 +74,7 @@ public class ZipEntryIterator implements Iterator<ZipEntryData>, AutoCloseable {
double compressionRatio = (double) totalSizeEntry / nextEntry.getCompressedSize();
if (compressionRatio > compressionThresholdRatio) {
throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio );
throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio);
}
}

26
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.service.download;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@ -21,6 +22,8 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetai
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateExportService;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -29,12 +32,13 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.download
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadRedactionFileStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
@ -68,6 +72,8 @@ public class DownloadPreparationService {
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository;
private final WebsocketService websocketService;
private final DossierPersistenceService dossierPersistenceService;
private final DossierTemplateExportService dossierTemplateExportService;
@Value("${storage.backend}")
private String storageBackend;
@ -134,7 +140,7 @@ public class DownloadPreparationService {
for (var downloadFileType : downloadFileTypes) {
switch (downloadFileType) {
case REDACTED -> result.add(RedactionType.REDACTED);
case PREVIEW -> result.add(RedactionType.PREVIEW);
case PREVIEW, DATASET_EXPORT -> result.add(RedactionType.PREVIEW);
case OPTIMIZED_PREVIEW -> result.add(RedactionType.OPTIMIZED_PREVIEW);
case DELTA_PREVIEW -> result.add(RedactionType.DELTA);
default -> {
@ -198,7 +204,7 @@ public class DownloadPreparationService {
var redactionFileResults = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(downloadId);
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadId);
var storedFileInformations = getStoredFileInformation(downloadId);
List<StoredFileInformation> storedFileInformations = getStoredFileInformation(downloadId);
try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {
@ -276,6 +282,11 @@ public class DownloadPreparationService {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(file.getFilename(), "redacted"), //
getRedacted(file.getId(), redactionFileResult.get().getDetails())));
}
if (downloadFileType.equals(DownloadFileType.DATASET_EXPORT)) {
String dossierName = dossierPersistenceService.getDossierNameById(file.getDossierId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierName, file.getFilename(),//
getPreview(file.getId(), redactionFileResult.get().getDetails())));
}
}
log.info("Successfully added file {}/{} for downloadId {}, took {}",
@ -285,6 +296,10 @@ public class DownloadPreparationService {
System.currentTimeMillis() - start);
i++;
}
if (downloadStatus.getDownloadFileTypes().contains(DownloadFileType.DATASET_EXPORT)) {
DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(downloadStatus.getDossier().getDossierTemplateId());
dossierTemplateExportService.addDossierTemplateToArchive(fileSystemBackedArchiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate);
}
log.info("Successfully added {} files for downloadId {}, took {}",
downloadStatus.getFiles()
.stream()
@ -381,7 +396,10 @@ public class DownloadPreparationService {
} else {
storageId = generateReportJsonStorageIdForAzure(downloadId);
}
if (!fileManagementStorageService.objectExists(storageId)) {
log.warn("Could not find stored file information with id {}", storageId);
return new ArrayList<>();
}
return objectMapper.readValue(fileManagementStorageService.getStoredObjectBytes(storageId), new TypeReference<>() {
});
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java
View File

@ -22,8 +22,10 @@ public class DownloadReportCleanupService {
public void deleteTmpReportFiles(Collection<String> storageIds) {
for (String storageId : storageIds) {
fileManagementStorageService.deleteObject(storageId);
log.info("Deleted tmp report file {}", storageId);
if (fileManagementStorageService.objectExists(storageId)) {
fileManagementStorageService.deleteObject(storageId);
log.info("Deleted tmp report file {}", storageId);
}
}
}

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java
View File

@ -209,7 +209,7 @@ public class DossierPersistenceService {
@Transactional
public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) {
public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) {
dossierRepository.hardDelete(dossierId, hardDeletedTime, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
@ -275,4 +275,11 @@ public class DossierPersistenceService {
return dossierRepository.findDossierTemplateId(dossierId);
}
public String getDossierNameById(String dossierId) {
return dossierRepository.findDossierNameById(dossierId)
.orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
}
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java
View File

@ -38,7 +38,7 @@ public class DownloadStatusPersistenceService {
}
public void createStatus(String userId,
public DownloadStatusEntity createStatus(String userId,
String storageId,
DossierEntity dossier,
String filename,
@ -62,7 +62,7 @@ public class DownloadStatusPersistenceService {
downloadStatus.setRedactionPreviewColor(redactionPreviewColor);
downloadStatus.setStatus(DownloadStatusValue.QUEUED);
downloadStatusRepository.save(downloadStatus);
return downloadStatusRepository.save(downloadStatus);
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.springframework.data.jpa.repository.JpaRepository;
@ -110,4 +111,8 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
@Query("SELECT d FROM DossierEntity d WHERE d.hardDeletedTime IS NULL AND d.softDeletedTime IS NOT NULL AND d.softDeletedTime < :deletionThreshold")
List<DossierEntity> findDossiersSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold);
@Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL")
Optional<String> findDossierNameById(@Param("dossierId") String dossierId);
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java
View File

@ -6,6 +6,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
@ -50,7 +51,7 @@ public class FileSystemBackedArchiver implements AutoCloseable {
this.rethrowExceptions = rethrowExceptions;
tempFile = FileUtils.createTempFile("archive", ".zip");
zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile));
zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile), StandardCharsets.UTF_8);
}

4
persistence-service-v1/persistence-service-server-v1/build.gradle.kts
View File

@ -41,19 +41,19 @@ description = "persistence-service-server-v1"
tasks.named<BootBuildImage>("bootBuildImage") {
builder.set("docker-proxy.knecon.com/paketobuildpacks/builder:base")
runImage.set("docker-proxy.knecon.com/paketobuildpacks/run:base-cnb")
environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // FileInputStream or ZipInputStream does not seem to care for file.encoding
imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}")
if (project.hasProperty("buildbootDockerHostNetwork")) {
network.set("host")
}
docker {
docker {
if (project.hasProperty("buildbootDockerHostNetwork")) {
bindHostToBuilder.set(true)
}

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java
View File

@ -7,5 +7,6 @@ public enum DownloadFileType {
REDACTED,
ANNOTATED,
FLATTEN,
DELTA_PREVIEW
DELTA_PREVIEW,
DATASET_EXPORT
}