RED-10127: Import/Export of dossiertemplate with preview files

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
@@ -48,6 +49,7 @@ public class SupportController implements SupportResource {
     private final FileStatusManagementService fileStatusManagementService;
     private final FileExchangeExportService fileExchangeExportService;
     private final FileExchangeImportService fileExchangeImportService;
+    private final DatasetExchangeService datasetExchangeService;
 
 
     @Override
@@ -127,6 +129,13 @@ public class SupportController implements SupportResource {
     }
 
 
+    @Override
+    public DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
+
+        return datasetExchangeService.prepareExport(dossierTemplateId);
+    }
+
+
     @Override
     public DownloadResponse exportFiles(String dossierTemplateId, FileExchangeExportRequest exportRequest) {
 
@@ -147,4 +156,17 @@ public class SupportController implements SupportResource {
         return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
     }
 
+    @Override
+    @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
+    public ImportResponse importDataset(MultipartFile file) {
+
+        byte[] bytes;
+        try {
+            bytes = file.getBytes();
+        } catch (IOException e) {
+            throw new BadRequestException("File could not be read and is likely corrupted.", e);
+        }
+        return datasetExchangeService.importDataset(KeycloakSecurity.getUserId(), bytes);
+    }
+
 }

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -34,6 +34,7 @@ public interface SupportResource {
 
     String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status/filter";
     String FILE_EXCHANGE_REST_PATH = ExternalApi.BASE_PATH + "/file-exchange";
+    String DATASET_EXCHANGE = ExternalApi.BASE_PATH + "/dataset-exchange";
 
     String BULK_REST_PATH = "/bulk";
 
@@ -143,6 +144,21 @@ public interface SupportResource {
     DownloadResponse exportFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody FileExchangeExportRequest exportRequest);
 
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = DATASET_EXCHANGE
+                         + EXPORT
+                         + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
+            ## Export Preview Files Endpoint
+            
+            Use this endpoint to export all files of a DossierTemplate as a Preview file containing all applied annotations as redaction annotations with additional data
+            The endpoint returns a String storageId, which is used to query the DownloadController for the export zip archive's status and to download the archive.
+            """)
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId);
+
+
     @ResponseBody
     @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@@ -150,4 +166,11 @@ public interface SupportResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
     ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
+    @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
+    @PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. Returns the resulting dossierTemplateId.")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    ImportResponse importDataset(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java

@@ -0,0 +1,57 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DatasetExchangeImportModel {
+
+    private ImportTemplateResult importTemplateResult = ImportTemplateResult.builder().build();
+    private List<Dossier> dossiers = new LinkedList<>();
+
+    @Builder
+    @Getter
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    public static class Dossier {
+
+        String name;
+        @Builder.Default
+        List<File> files = new LinkedList<>();
+
+    }
+
+    @Getter
+    @AllArgsConstructor
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    public static class File {
+
+        String name;
+        byte[] data;
+
+    }
+
+
+    public DatasetExchangeImportModel.Dossier getOrCreateDossier(String name) {
+
+        for (Dossier dossier : dossiers) {
+            if (dossier.getName().equals(name)) {
+                return dossier;
+            }
+        }
+        Dossier dossier = Dossier.builder().name(name).build();
+        dossiers.add(dossier);
+        return dossier;
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java

@@ -0,0 +1,183 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import static com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeArchivalService.SIZE_THRESHOLD;
+
+import java.time.OffsetDateTime;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
+import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
+import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DatasetExchangeArchiveReader;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
+import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
+import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadPreparationService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DatasetExchangeService {
+
+    DossierTemplateManagementService dossierTemplateManagementService;
+    DownloadStatusPersistenceService downloadStatusPersistenceService;
+    FileStatusPersistenceService fileStatusPersistenceService;
+    DossierPersistenceService dossierPersistenceService;
+    ColorsService colorsService;
+    RabbitTemplate rabbitTemplate;
+    FileManagementServiceSettings settings;
+    DossierTemplateImportService dossierTemplateImportService;
+    DossierCreatorService dossierCreatorService;
+    ReportTemplatePersistenceService reportTemplateService;
+    UploadService uploadService;
+    DownloadPreparationService downloadPreparationService;
+    FileManagementStorageService storageService;
+
+
+    public DownloadResponse prepareExport(String dossierTemplateId) {
+
+        var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId);
+        String downloadFilename = template.getName() + "_PREVIEW_FILES.zip";
+        String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES");
+        List<FileEntity> files = fileStatusPersistenceService.getStatusesForDossierTemplate(dossierTemplateId);
+        List<DossierEntity> dossiers = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
+        if (dossiers.isEmpty()) {
+            throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId);
+        }
+        ColorsEntity colors = colorsService.getColors(dossierTemplateId);
+
+        downloadPreparationService.clearRedactionStatusEntries(storageId);
+        if (storageService.objectExists(storageId)) {
+            storageService.deleteObject(storageId);
+        }
+
+        DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.createStatus(KeycloakSecurity.getUserId(),
+                                                                                            storageId,
+                                                                                            dossiers.get(0),
+                                                                                            downloadFilename,
+                                                                                            "application/zip",
+                                                                                            files.stream()
+                                                                                                    .map(FileEntity::getId)
+                                                                                                    .toList(),
+                                                                                            Set.of(DownloadFileType.DATASET_EXPORT),
+                                                                                            Collections.emptyList(),
+                                                                                            colors.getPreviewColor());
+
+        RedactionMessage.RedactionMessageBuilder builder = RedactionMessage.builder()
+                .appliedRedactionColor(colors.getRedactionColor())
+                .redactionPreviewColor(colors.getPreviewColor())
+                .keepHiddenText(true)
+                .keepOverlappingObjects(true)
+                .keepImageMetaData(true)
+                .downloadId(storageId)
+                .redactionTypes(List.of(RedactionType.PREVIEW));
+
+        downloadStatus.getFiles()
+                .forEach(fileEntity -> {
+                    RedactionMessage message = builder.dossierId(fileEntity.getDossierId())
+                            .fileId(fileEntity.getId())
+                            .unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED)
+                            .build();
+                    log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", storageId, fileEntity.getId());
+                    rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_REQUEST_EXCHANGE, TenantContext.getTenantId(), message);
+                });
+
+        downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.GENERATING);
+
+        return new DownloadResponse(storageId);
+    }
+
+
+    @SneakyThrows
+    public ImportResponse importDataset(String userId, byte[] archive) {
+
+        DatasetExchangeArchiveReader datasetExchangeArchiveReader = new DatasetExchangeArchiveReader(userId);
+        try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
+            zipEntryIterator.forEachRemaining(datasetExchangeArchiveReader::handleZipEntryData);
+        }
+        DatasetExchangeImportModel importModel = datasetExchangeArchiveReader.getDatasetExchangeImportModel();
+
+        TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importModel.getImportTemplateResult());
+        for (DatasetExchangeImportModel.Dossier dossier : importModel.getDossiers()) {
+
+            String dossierId = getDossierForImport(templateImportInfo, dossier, userId);
+            for (DatasetExchangeImportModel.File file : dossier.getFiles()) {
+                uploadService.processSingleFile(dossierId, file.getName(), file.getData(), false, false);
+            }
+        }
+        return new ImportResponse(templateImportInfo.getDossierTemplateId());
+    }
+
+
+    private String getDossierForImport(TemplateImportInfo templateImportInfo, DatasetExchangeImportModel.Dossier dossier, String userId) {
+
+        String dossierTemplateId = templateImportInfo.getDossierTemplateId();
+
+        CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
+                .dossierName(dossier.getName())
+                .description(dossier.getName())
+                .downloadFileTypes(Collections.emptySet())
+                .dossierTemplateId(dossierTemplateId)
+                .requestingUser(userId)
+                .reportTemplateIds(reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
+                                           .stream()
+                                           .map(ReportTemplateEntity::getTemplateId)
+                                           .toList())
+                .build();
+
+        Dossier importedDossier = null;
+        int retries = 0;
+        while (importedDossier == null && retries < 100) {
+            try {
+                importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
+            } catch (ConflictException e) {
+                retries++;
+                request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
+            }
+        }
+        if (importedDossier == null) {
+            throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
+        }
+        return importedDossier.getId();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java

@@ -67,7 +67,7 @@ public class DossierImportService {
 
         Dossier importedDossier = null;
         int retries = 0;
-        while (importedDossier == null && retries < 10) {
+        while (importedDossier == null && retries < 100) {
             try {
                 importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
             } catch (ConflictException e) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java

@@ -72,6 +72,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -150,8 +151,9 @@ public class DossierTemplateExportService {
     }
 
 
+    @SneakyThrows
     @Observed(name = "DossierTemplateExportService", contextualName = "write-dossier-template-to-archive")
-    public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) throws IOException {
+    public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) {
 
         // add dossier template name and meta data
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java

@@ -0,0 +1,50 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders;
+
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel;
+
+import lombok.AccessLevel;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DatasetExchangeArchiveReader {
+
+    DossierTemplateArchiveReader dossierTemplateArchiveReader;
+    DatasetExchangeImportModel datasetExchangeImportModel;
+
+
+    public DatasetExchangeArchiveReader(String userId) {
+
+        dossierTemplateArchiveReader = new DossierTemplateArchiveReader(userId, null, false);
+        datasetExchangeImportModel = new DatasetExchangeImportModel();
+    }
+
+
+    public DatasetExchangeImportModel getDatasetExchangeImportModel() {
+
+        datasetExchangeImportModel.setImportTemplateResult(dossierTemplateArchiveReader.buildResult());
+        return datasetExchangeImportModel;
+    }
+
+
+    @SneakyThrows
+    public void handleZipEntryData(ZipEntryData zipEntryData) {
+
+        if (zipEntryData.getBaseFolder().equals(FileExchangeNames.TEMPLATE_FOLDER)) {
+            dossierTemplateArchiveReader.handleZipEntryData(zipEntryData);
+        } else if (zipEntryData.getDepth() >= 1) { // in dossier folders
+
+            String dossierName = zipEntryData.getBaseFolder().replaceAll("\\s*\\([1-9]+\\)", "");
+            String fileName = zipEntryData.getFileName();
+            if (!fileName.endsWith(".pdf")) {
+                return;
+            }
+
+            var dossier = datasetExchangeImportModel.getOrCreateDossier(dossierName);
+            dossier.getFiles().add(new DatasetExchangeImportModel.File(fileName, zipEntryData.data()));
+        }
+
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders;
 
-import java.io.*;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 import java.util.UUID;
@@ -70,7 +74,7 @@ public class ZipEntryIterator implements Iterator<ZipEntryData>, AutoCloseable {
 
                 double compressionRatio = (double) totalSizeEntry / nextEntry.getCompressedSize();
                 if (compressionRatio > compressionThresholdRatio) {
-                    throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio );
+                    throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio);
                 }
             }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.download;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
@@ -21,6 +22,8 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetai
 import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateExportService;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@@ -29,12 +32,13 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.download
 import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
-import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadRedactionFileStatusRepository;
+import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
 import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
@@ -68,6 +72,8 @@ public class DownloadPreparationService {
     private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
     private final DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository;
     private final WebsocketService websocketService;
+    private final DossierPersistenceService dossierPersistenceService;
+    private final DossierTemplateExportService dossierTemplateExportService;
 
     @Value("${storage.backend}")
     private String storageBackend;
@@ -134,7 +140,7 @@ public class DownloadPreparationService {
         for (var downloadFileType : downloadFileTypes) {
             switch (downloadFileType) {
                 case REDACTED -> result.add(RedactionType.REDACTED);
-                case PREVIEW -> result.add(RedactionType.PREVIEW);
+                case PREVIEW, DATASET_EXPORT -> result.add(RedactionType.PREVIEW);
                 case OPTIMIZED_PREVIEW -> result.add(RedactionType.OPTIMIZED_PREVIEW);
                 case DELTA_PREVIEW -> result.add(RedactionType.DELTA);
                 default -> {
@@ -198,7 +204,7 @@ public class DownloadPreparationService {
         var redactionFileResults = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(downloadId);
         DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadId);
 
-        var storedFileInformations = getStoredFileInformation(downloadId);
+        List<StoredFileInformation> storedFileInformations = getStoredFileInformation(downloadId);
 
         try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {
 
@@ -276,6 +282,11 @@ public class DownloadPreparationService {
                     fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(file.getFilename(), "redacted"), //
                                                                                                 getRedacted(file.getId(), redactionFileResult.get().getDetails())));
                 }
+                if (downloadFileType.equals(DownloadFileType.DATASET_EXPORT)) {
+                    String dossierName = dossierPersistenceService.getDossierNameById(file.getDossierId());
+                    fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierName, file.getFilename(),//
+                                                                                                getPreview(file.getId(), redactionFileResult.get().getDetails())));
+                }
 
             }
             log.info("Successfully added file {}/{} for downloadId {}, took {}",
@@ -285,6 +296,10 @@ public class DownloadPreparationService {
                      System.currentTimeMillis() - start);
             i++;
         }
+        if (downloadStatus.getDownloadFileTypes().contains(DownloadFileType.DATASET_EXPORT)) {
+            DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(downloadStatus.getDossier().getDossierTemplateId());
+            dossierTemplateExportService.addDossierTemplateToArchive(fileSystemBackedArchiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate);
+        }
         log.info("Successfully added {} files for downloadId {}, took {}",
                  downloadStatus.getFiles()
                          .stream()
@@ -381,7 +396,10 @@ public class DownloadPreparationService {
         } else {
             storageId = generateReportJsonStorageIdForAzure(downloadId);
         }
-
+        if (!fileManagementStorageService.objectExists(storageId)) {
+            log.warn("Could not find stored file information with id {}", storageId);
+            return new ArrayList<>();
+        }
         return objectMapper.readValue(fileManagementStorageService.getStoredObjectBytes(storageId), new TypeReference<>() {
         });
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java

@@ -22,8 +22,10 @@ public class DownloadReportCleanupService {
     public void deleteTmpReportFiles(Collection<String> storageIds) {
 
         for (String storageId : storageIds) {
-            fileManagementStorageService.deleteObject(storageId);
+            if (fileManagementStorageService.objectExists(storageId)) {
-            log.info("Deleted tmp report file {}", storageId);
+                fileManagementStorageService.deleteObject(storageId);
+                log.info("Deleted tmp report file {}", storageId);
+            }
         }
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java

@@ -209,7 +209,7 @@ public class DossierPersistenceService {
 
 
     @Transactional
-    public void hardDelete(String dossierId,  OffsetDateTime hardDeletedTime) {
+    public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) {
 
         dossierRepository.hardDelete(dossierId, hardDeletedTime, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
     }
@@ -275,4 +275,11 @@ public class DossierPersistenceService {
         return dossierRepository.findDossierTemplateId(dossierId);
     }
 
+
+    public String getDossierNameById(String dossierId) {
+
+        return dossierRepository.findDossierNameById(dossierId)
+                .orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java

@@ -38,7 +38,7 @@ public class DownloadStatusPersistenceService {
     }
 
 
-    public void createStatus(String userId,
+    public DownloadStatusEntity createStatus(String userId,
                              String storageId,
                              DossierEntity dossier,
                              String filename,
@@ -62,7 +62,7 @@ public class DownloadStatusPersistenceService {
         downloadStatus.setRedactionPreviewColor(redactionPreviewColor);
         downloadStatus.setStatus(DownloadStatusValue.QUEUED);
 
-        downloadStatusRepository.save(downloadStatus);
+        return downloadStatusRepository.save(downloadStatus);
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
 
 import java.time.OffsetDateTime;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 
 import org.springframework.data.jpa.repository.JpaRepository;
@@ -110,4 +111,8 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
     @Query("SELECT d FROM DossierEntity d WHERE d.hardDeletedTime IS NULL AND d.softDeletedTime IS NOT NULL AND d.softDeletedTime < :deletionThreshold")
     List<DossierEntity> findDossiersSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold);
 
+
+    @Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL")
+    Optional<String> findDossierNameById(@Param("dossierId") String dossierId);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java

@@ -6,6 +6,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
@@ -50,7 +51,7 @@ public class FileSystemBackedArchiver implements AutoCloseable {
 
         this.rethrowExceptions = rethrowExceptions;
         tempFile = FileUtils.createTempFile("archive", ".zip");
-        zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile));
+        zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile), StandardCharsets.UTF_8);
     }
 
 

persistence-service-v1/persistence-service-server-v1/build.gradle.kts

@@ -41,19 +41,19 @@ description = "persistence-service-server-v1"
 
 tasks.named<BootBuildImage>("bootBuildImage") {
 
-
     builder.set("docker-proxy.knecon.com/paketobuildpacks/builder:base")
     runImage.set("docker-proxy.knecon.com/paketobuildpacks/run:base-cnb")
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
     environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
+    environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // FileInputStream or ZipInputStream does not seem to care for file.encoding
 
     imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}")
     if (project.hasProperty("buildbootDockerHostNetwork")) {
         network.set("host")
     }
-    docker {
 
+    docker {
         if (project.hasProperty("buildbootDockerHostNetwork")) {
             bindHostToBuilder.set(true)
         }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java

@@ -7,5 +7,6 @@ public enum DownloadFileType {
     REDACTED,
     ANNOTATED,
     FLATTEN,
-    DELTA_PREVIEW
+    DELTA_PREVIEW,
+    DATASET_EXPORT
 }