From c52dfbdf5abb056940172c4dab12408c546bda2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Mon, 14 Oct 2024 15:30:18 +0200 Subject: [PATCH] RED-10127: Import/Export of dossiertemplate with preview files --- .../impl/controller/SupportController.java | 22 +++ .../external/resource/SupportResource.java | 23 +++ .../models/DatasetExchangeImportModel.java | 57 ++++++ .../service/DatasetExchangeService.java | 183 ++++++++++++++++++ .../service/DossierImportService.java | 2 +- .../service/DossierTemplateExportService.java | 4 +- .../DatasetExchangeArchiveReader.java | 50 +++++ .../zipreaders/ZipEntryIterator.java | 8 +- .../download/DownloadPreparationService.java | 26 ++- .../DownloadReportCleanupService.java | 6 +- .../DossierPersistenceService.java | 9 +- .../DownloadStatusPersistenceService.java | 4 +- .../repository/DossierRepository.java | 5 + .../utils/FileSystemBackedArchiver.java | 3 +- .../build.gradle.kts | 4 +- .../dossiertemplate/DownloadFileType.java | 3 +- 16 files changed, 392 insertions(+), 17 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java index 66a222fcd..e0b802fce 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java @@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService; @@ -48,6 +49,7 @@ public class SupportController implements SupportResource { private final FileStatusManagementService fileStatusManagementService; private final FileExchangeExportService fileExchangeExportService; private final FileExchangeImportService fileExchangeImportService; + private final DatasetExchangeService datasetExchangeService; @Override @@ -127,6 +129,13 @@ public class SupportController implements SupportResource { } + @Override + public DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) { + + return datasetExchangeService.prepareExport(dossierTemplateId); + } + + @Override public DownloadResponse exportFiles(String dossierTemplateId, FileExchangeExportRequest exportRequest) { @@ -147,4 +156,17 @@ public class SupportController implements SupportResource { return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes); } + @Override + @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')") + public ImportResponse importDataset(MultipartFile file) { + + byte[] bytes; + try { + bytes = file.getBytes(); + } catch (IOException e) { + throw new BadRequestException("File could not be read and is likely corrupted.", e); + } + return datasetExchangeService.importDataset(KeycloakSecurity.getUserId(), bytes); + } + } diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index b8723183d..5782e030f 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -34,6 +34,7 @@ public interface SupportResource { String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status/filter"; String FILE_EXCHANGE_REST_PATH = ExternalApi.BASE_PATH + "/file-exchange"; + String DATASET_EXCHANGE = ExternalApi.BASE_PATH + "/dataset-exchange"; String BULK_REST_PATH = "/bulk"; @@ -143,6 +144,21 @@ public interface SupportResource { DownloadResponse exportFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody FileExchangeExportRequest exportRequest); + @ResponseStatus(value = HttpStatus.OK) + @ResponseBody + @PostMapping(value = DATASET_EXCHANGE + + EXPORT + + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """ + ## Export Preview Files Endpoint + + Use this endpoint to export all files of a DossierTemplate as a Preview file containing all applied annotations as redaction annotations with additional data + The endpoint returns a String storageId, which is used to query the DownloadController for the export zip archive's status and to download the archive. + """) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) + DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId); + + @ResponseBody @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) @@ -150,4 +166,11 @@ public interface SupportResource { @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file); + @ResponseBody + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. Returns the resulting dossierTemplateId.") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) + ImportResponse importDataset(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java new file mode 100644 index 000000000..a447758df --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java @@ -0,0 +1,57 @@ +package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models; + +import java.util.LinkedList; +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DatasetExchangeImportModel { + + private ImportTemplateResult importTemplateResult = ImportTemplateResult.builder().build(); + private List dossiers = new LinkedList<>(); + + @Builder + @Getter + @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) + public static class Dossier { + + String name; + @Builder.Default + List files = new LinkedList<>(); + + } + + @Getter + @AllArgsConstructor + @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) + public static class File { + + String name; + byte[] data; + + } + + + public DatasetExchangeImportModel.Dossier getOrCreateDossier(String name) { + + for (Dossier dossier : dossiers) { + if (dossier.getName().equals(name)) { + return dossier; + } + } + Dossier dossier = Dossier.builder().name(name).build(); + dossiers.add(dossier); + return dossier; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java new file mode 100644 index 000000000..ab765e3c6 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java @@ -0,0 +1,183 @@ +package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service; + +import static com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeArchivalService.SIZE_THRESHOLD; + +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType; +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DatasetExchangeArchiveReader; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.UploadService; +import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadPreparationService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; +import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; +import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DatasetExchangeService { + + DossierTemplateManagementService dossierTemplateManagementService; + DownloadStatusPersistenceService downloadStatusPersistenceService; + FileStatusPersistenceService fileStatusPersistenceService; + DossierPersistenceService dossierPersistenceService; + ColorsService colorsService; + RabbitTemplate rabbitTemplate; + FileManagementServiceSettings settings; + DossierTemplateImportService dossierTemplateImportService; + DossierCreatorService dossierCreatorService; + ReportTemplatePersistenceService reportTemplateService; + UploadService uploadService; + DownloadPreparationService downloadPreparationService; + FileManagementStorageService storageService; + + + public DownloadResponse prepareExport(String dossierTemplateId) { + + var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId); + String downloadFilename = template.getName() + "_PREVIEW_FILES.zip"; + String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES"); + List files = fileStatusPersistenceService.getStatusesForDossierTemplate(dossierTemplateId); + List dossiers = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId); + if (dossiers.isEmpty()) { + throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId); + } + ColorsEntity colors = colorsService.getColors(dossierTemplateId); + + downloadPreparationService.clearRedactionStatusEntries(storageId); + if (storageService.objectExists(storageId)) { + storageService.deleteObject(storageId); + } + + DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.createStatus(KeycloakSecurity.getUserId(), + storageId, + dossiers.get(0), + downloadFilename, + "application/zip", + files.stream() + .map(FileEntity::getId) + .toList(), + Set.of(DownloadFileType.DATASET_EXPORT), + Collections.emptyList(), + colors.getPreviewColor()); + + RedactionMessage.RedactionMessageBuilder builder = RedactionMessage.builder() + .appliedRedactionColor(colors.getRedactionColor()) + .redactionPreviewColor(colors.getPreviewColor()) + .keepHiddenText(true) + .keepOverlappingObjects(true) + .keepImageMetaData(true) + .downloadId(storageId) + .redactionTypes(List.of(RedactionType.PREVIEW)); + + downloadStatus.getFiles() + .forEach(fileEntity -> { + RedactionMessage message = builder.dossierId(fileEntity.getDossierId()) + .fileId(fileEntity.getId()) + .unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED) + .build(); + log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", storageId, fileEntity.getId()); + rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_REQUEST_EXCHANGE, TenantContext.getTenantId(), message); + }); + + downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.GENERATING); + + return new DownloadResponse(storageId); + } + + + @SneakyThrows + public ImportResponse importDataset(String userId, byte[] archive) { + + DatasetExchangeArchiveReader datasetExchangeArchiveReader = new DatasetExchangeArchiveReader(userId); + try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) { + zipEntryIterator.forEachRemaining(datasetExchangeArchiveReader::handleZipEntryData); + } + DatasetExchangeImportModel importModel = datasetExchangeArchiveReader.getDatasetExchangeImportModel(); + + TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importModel.getImportTemplateResult()); + for (DatasetExchangeImportModel.Dossier dossier : importModel.getDossiers()) { + + String dossierId = getDossierForImport(templateImportInfo, dossier, userId); + for (DatasetExchangeImportModel.File file : dossier.getFiles()) { + uploadService.processSingleFile(dossierId, file.getName(), file.getData(), false, false); + } + } + return new ImportResponse(templateImportInfo.getDossierTemplateId()); + } + + + private String getDossierForImport(TemplateImportInfo templateImportInfo, DatasetExchangeImportModel.Dossier dossier, String userId) { + + String dossierTemplateId = templateImportInfo.getDossierTemplateId(); + + CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder() + .dossierName(dossier.getName()) + .description(dossier.getName()) + .downloadFileTypes(Collections.emptySet()) + .dossierTemplateId(dossierTemplateId) + .requestingUser(userId) + .reportTemplateIds(reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId()) + .stream() + .map(ReportTemplateEntity::getTemplateId) + .toList()) + .build(); + + Dossier importedDossier = null; + int retries = 0; + while (importedDossier == null && retries < 100) { + try { + importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId); + } catch (ConflictException e) { + retries++; + request.setDossierName(String.format("%s (%d)", dossier.getName(), retries)); + } + } + if (importedDossier == null) { + throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries)); + } + return importedDossier.getId(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java index f309674e3..bf1ce59e0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java @@ -67,7 +67,7 @@ public class DossierImportService { Dossier importedDossier = null; int retries = 0; - while (importedDossier == null && retries < 10) { + while (importedDossier == null && retries < 100) { try { importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId); } catch (ConflictException e) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java index 61a56b934..03d2b8ef7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java @@ -72,6 +72,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext; import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @@ -150,8 +151,9 @@ public class DossierTemplateExportService { } + @SneakyThrows @Observed(name = "DossierTemplateExportService", contextualName = "write-dossier-template-to-archive") - public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) throws IOException { + public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) { // add dossier template name and meta data fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java new file mode 100644 index 000000000..97291063a --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java @@ -0,0 +1,50 @@ +package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders; + +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel; + +import lombok.AccessLevel; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; + +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DatasetExchangeArchiveReader { + + DossierTemplateArchiveReader dossierTemplateArchiveReader; + DatasetExchangeImportModel datasetExchangeImportModel; + + + public DatasetExchangeArchiveReader(String userId) { + + dossierTemplateArchiveReader = new DossierTemplateArchiveReader(userId, null, false); + datasetExchangeImportModel = new DatasetExchangeImportModel(); + } + + + public DatasetExchangeImportModel getDatasetExchangeImportModel() { + + datasetExchangeImportModel.setImportTemplateResult(dossierTemplateArchiveReader.buildResult()); + return datasetExchangeImportModel; + } + + + @SneakyThrows + public void handleZipEntryData(ZipEntryData zipEntryData) { + + if (zipEntryData.getBaseFolder().equals(FileExchangeNames.TEMPLATE_FOLDER)) { + dossierTemplateArchiveReader.handleZipEntryData(zipEntryData); + } else if (zipEntryData.getDepth() >= 1) { // in dossier folders + + String dossierName = zipEntryData.getBaseFolder().replaceAll("\\s*\\([1-9]+\\)", ""); + String fileName = zipEntryData.getFileName(); + if (!fileName.endsWith(".pdf")) { + return; + } + + var dossier = datasetExchangeImportModel.getOrCreateDossier(dossierName); + dossier.getFiles().add(new DatasetExchangeImportModel.File(fileName, zipEntryData.data())); + } + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java index 2669501dd..506b09209 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java @@ -1,6 +1,10 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders; -import java.io.*; +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.UUID; @@ -70,7 +74,7 @@ public class ZipEntryIterator implements Iterator, AutoCloseable { double compressionRatio = (double) totalSizeEntry / nextEntry.getCompressedSize(); if (compressionRatio > compressionThresholdRatio) { - throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio ); + throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java index 6e6b2050f..f45a7c38b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java @@ -1,6 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.download; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -21,6 +22,8 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetai import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateExportService; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; @@ -29,12 +32,13 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.download import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadRedactionFileStatusRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver; import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest; @@ -68,6 +72,8 @@ public class DownloadPreparationService { private final DossierTemplatePersistenceService dossierTemplatePersistenceService; private final DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository; private final WebsocketService websocketService; + private final DossierPersistenceService dossierPersistenceService; + private final DossierTemplateExportService dossierTemplateExportService; @Value("${storage.backend}") private String storageBackend; @@ -134,7 +140,7 @@ public class DownloadPreparationService { for (var downloadFileType : downloadFileTypes) { switch (downloadFileType) { case REDACTED -> result.add(RedactionType.REDACTED); - case PREVIEW -> result.add(RedactionType.PREVIEW); + case PREVIEW, DATASET_EXPORT -> result.add(RedactionType.PREVIEW); case OPTIMIZED_PREVIEW -> result.add(RedactionType.OPTIMIZED_PREVIEW); case DELTA_PREVIEW -> result.add(RedactionType.DELTA); default -> { @@ -198,7 +204,7 @@ public class DownloadPreparationService { var redactionFileResults = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(downloadId); DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadId); - var storedFileInformations = getStoredFileInformation(downloadId); + List storedFileInformations = getStoredFileInformation(downloadId); try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) { @@ -276,6 +282,11 @@ public class DownloadPreparationService { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(file.getFilename(), "redacted"), // getRedacted(file.getId(), redactionFileResult.get().getDetails()))); } + if (downloadFileType.equals(DownloadFileType.DATASET_EXPORT)) { + String dossierName = dossierPersistenceService.getDossierNameById(file.getDossierId()); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierName, file.getFilename(),// + getPreview(file.getId(), redactionFileResult.get().getDetails()))); + } } log.info("Successfully added file {}/{} for downloadId {}, took {}", @@ -285,6 +296,10 @@ public class DownloadPreparationService { System.currentTimeMillis() - start); i++; } + if (downloadStatus.getDownloadFileTypes().contains(DownloadFileType.DATASET_EXPORT)) { + DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(downloadStatus.getDossier().getDossierTemplateId()); + dossierTemplateExportService.addDossierTemplateToArchive(fileSystemBackedArchiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate); + } log.info("Successfully added {} files for downloadId {}, took {}", downloadStatus.getFiles() .stream() @@ -381,7 +396,10 @@ public class DownloadPreparationService { } else { storageId = generateReportJsonStorageIdForAzure(downloadId); } - + if (!fileManagementStorageService.objectExists(storageId)) { + log.warn("Could not find stored file information with id {}", storageId); + return new ArrayList<>(); + } return objectMapper.readValue(fileManagementStorageService.getStoredObjectBytes(storageId), new TypeReference<>() { }); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java index c2a69b616..3c9b9ebb9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java @@ -22,8 +22,10 @@ public class DownloadReportCleanupService { public void deleteTmpReportFiles(Collection storageIds) { for (String storageId : storageIds) { - fileManagementStorageService.deleteObject(storageId); - log.info("Deleted tmp report file {}", storageId); + if (fileManagementStorageService.objectExists(storageId)) { + fileManagementStorageService.deleteObject(storageId); + log.info("Deleted tmp report file {}", storageId); + } } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index 21a03fb26..6f8f21c69 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -209,7 +209,7 @@ public class DossierPersistenceService { @Transactional - public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) { + public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) { dossierRepository.hardDelete(dossierId, hardDeletedTime, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } @@ -275,4 +275,11 @@ public class DossierPersistenceService { return dossierRepository.findDossierTemplateId(dossierId); } + + public String getDossierNameById(String dossierId) { + + return dossierRepository.findDossierNameById(dossierId) + .orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE)); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java index a9ec3de3c..6dea74c67 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java @@ -38,7 +38,7 @@ public class DownloadStatusPersistenceService { } - public void createStatus(String userId, + public DownloadStatusEntity createStatus(String userId, String storageId, DossierEntity dossier, String filename, @@ -62,7 +62,7 @@ public class DownloadStatusPersistenceService { downloadStatus.setRedactionPreviewColor(redactionPreviewColor); downloadStatus.setStatus(DownloadStatusValue.QUEUED); - downloadStatusRepository.save(downloadStatus); + return downloadStatusRepository.save(downloadStatus); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index 438abb196..a844fc87a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis import java.time.OffsetDateTime; import java.util.List; +import java.util.Optional; import java.util.Set; import org.springframework.data.jpa.repository.JpaRepository; @@ -110,4 +111,8 @@ public interface DossierRepository extends JpaRepository @Query("SELECT d FROM DossierEntity d WHERE d.hardDeletedTime IS NULL AND d.softDeletedTime IS NOT NULL AND d.softDeletedTime < :deletionThreshold") List findDossiersSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold); + + @Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL") + Optional findDossierNameById(@Param("dossierId") String dossierId); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java index 1bb88f481..d2f354de2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java @@ -6,6 +6,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; @@ -50,7 +51,7 @@ public class FileSystemBackedArchiver implements AutoCloseable { this.rethrowExceptions = rethrowExceptions; tempFile = FileUtils.createTempFile("archive", ".zip"); - zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile)); + zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile), StandardCharsets.UTF_8); } diff --git a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts index 3a667d791..f03ed5f7a 100644 --- a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts @@ -41,19 +41,19 @@ description = "persistence-service-server-v1" tasks.named("bootBuildImage") { - builder.set("docker-proxy.knecon.com/paketobuildpacks/builder:base") runImage.set("docker-proxy.knecon.com/paketobuildpacks/run:base-cnb") environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ") environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8") + environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // FileInputStream or ZipInputStream does not seem to care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}") if (project.hasProperty("buildbootDockerHostNetwork")) { network.set("host") } - docker { + docker { if (project.hasProperty("buildbootDockerHostNetwork")) { bindHostToBuilder.set(true) } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java index f90cb5e37..e8cd137d6 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java @@ -7,5 +7,6 @@ public enum DownloadFileType { REDACTED, ANNOTATED, FLATTEN, - DELTA_PREVIEW + DELTA_PREVIEW, + DATASET_EXPORT }