hard delete dicts

RED-9740: Added migration to add graphic type

See merge request redactmanager/persistence-service!623

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -8,7 +8,7 @@ plugins {
 
 val redactionServiceVersion by rootProject.extra { "4.290.0" }
 val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" }
-val redactionReportServiceVersion by rootProject.extra { "4.64.0" }
+val redactionReportServiceVersion by rootProject.extra { "4.73.6" }
 val searchServiceVersion by rootProject.extra { "2.90.0" }
 
 repositories {

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java

@@ -381,7 +381,6 @@ public class DossierController implements DossierResource {
             throw new AccessDeniedException("Can not delete dossier that is owned by a different user");
         }
 
-
         dossierManagementService.softDeleteDossier(dossierId);
 
         auditPersistenceService.audit(AuditRequest.builder()
@@ -450,7 +449,7 @@ public class DossierController implements DossierResource {
     @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getSoftDeletedDossiers() {
 
-        var dossiers =  dossierManagementService.getSoftDeletedDossiers()
+        var dossiers = dossierManagementService.getSoftDeletedDossiers()
                 .stream()
                 .map(dossierACLService::enhanceDossierWithACLData)
                 .collect(Collectors.toList());
@@ -463,7 +462,7 @@ public class DossierController implements DossierResource {
     @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiers() {
 
-        var dossiers =  dossierManagementService.getArchivedDossiers()
+        var dossiers = dossierManagementService.getArchivedDossiers()
                 .stream()
                 .map(dossierACLService::enhanceDossierWithACLData)
                 .collect(Collectors.toList());

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierTemplateController.java

@@ -24,13 +24,15 @@ import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
 import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ExportDownloadRequest;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateExportService;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateImportService;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateStatsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.DossierTemplateResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
@@ -42,7 +44,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStats;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ExportDownloadRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
@@ -61,7 +62,8 @@ public class DossierTemplateController implements DossierTemplateResource {
     private final AuditPersistenceService auditPersistenceService;
     private final DossierManagementService dossierManagementService;
     private final DossierACLService dossierACLService;
-    private final UserService userService;
+    private final DossierTemplateImportService dossierTemplateImportService;
+    private final DossierTemplateExportService dossierTemplateExportService;
 
 
     @Override
@@ -125,7 +127,7 @@ public class DossierTemplateController implements DossierTemplateResource {
             throw new ConflictException("Can not delete dossier template because there are dossiers based on it");
         }
 
-        dossierTemplateManagementService.deleteDossierTemplate(dossierTemplateId, userId);
+        dossierTemplateManagementService.softDeleteDossierTemplate(dossierTemplateId, userId);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierTemplateId)
@@ -150,7 +152,7 @@ public class DossierTemplateController implements DossierTemplateResource {
                     throw new ConflictException("Can not delete dossier template because there are dossiers based on it");
                 }
 
-                dossierTemplateManagementService.deleteDossierTemplate(dossierTemplateId, userId);
+                dossierTemplateManagementService.softDeleteDossierTemplate(dossierTemplateId, userId);
                 auditPersistenceService.audit(AuditRequest.builder()
                                                       .userId(KeycloakSecurity.getUserId())
                                                       .objectId(dossierTemplateId)
@@ -224,15 +226,16 @@ public class DossierTemplateController implements DossierTemplateResource {
 
         try {
             ExportDownloadRequest request = ExportDownloadRequest.builder().dossierTemplateId(dossierTemplateId).userId(KeycloakSecurity.getUserId()).build();
-            var response = dossierTemplateManagementService.prepareExportDownload(request);
+            dossierTemplateManagementService.validateDossierTemplateForDuplicateRanks(request.getDossierTemplateId());
+            String storageId = dossierTemplateExportService.prepareExportDownload(request);
             auditPersistenceService.audit(AuditRequest.builder()
                                                   .userId(KeycloakSecurity.getUserId())
-                                                  .objectId(response.getValue())
+                                                  .objectId(storageId)
                                                   .category(AuditCategory.DOWNLOAD.name())
                                                   .message("Export Download was prepared")
                                                   .details(Map.of("dossierTemplateId", request.getDossierTemplateId()))
                                                   .build());
-            return new DownloadResponse(response.getValue());
+            return new DownloadResponse(storageId);
         } catch (FeignException e) {
             throw processFeignException(e);
         }
@@ -265,15 +268,15 @@ public class DossierTemplateController implements DossierTemplateResource {
                         .userId(KeycloakSecurity.getUserId())
                         .archive(file.getBytes())
                         .build();
-                DossierTemplate loadedDossierTemplate = dossierTemplateManagementService.importDossierTemplate(request);
+                String importedDossierTemplateId = dossierTemplateImportService.importDossierTemplate(request);
                 auditPersistenceService.audit(AuditRequest.builder()
                                                       .userId(KeycloakSecurity.getUserId())
-                                                      .objectId(loadedDossierTemplate.getId())
+                                                      .objectId(importedDossierTemplateId)
                                                       .category(AuditCategory.DOSSIER_TEMPLATE.name())
                                                       .message("Dossier template was imported")
-                                                      .details(Map.of("dossierTemplateId", loadedDossierTemplate.getId()))
+                                                      .details(Map.of("dossierTemplateId", importedDossierTemplateId))
                                                       .build());
-                return convert(loadedDossierTemplate);
+                return convert(dossierTemplateManagementService.getDossierTemplate(importedDossierTemplateId));
             } catch (IOException e) {
                 throw new BadRequestException(e.getMessage(), e);
             } catch (FeignException e) {

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/EntityLogController.java

@@ -37,7 +37,10 @@ public class EntityLogController implements EntityLogResource {
 
         accessControlService.checkViewPermissionsToDossier(dossierId);
         accessControlService.validateFileResourceExistence(fileId);
-        return entityLogResponseMapper.toLogResponse(entityLogService.getEntityLog(dossierId, fileId, excludedTypes == null ? new ArrayList<>() : excludedTypes, includeUnprocessed));
+        return entityLogResponseMapper.toLogResponse(entityLogService.getEntityLog(dossierId,
+                                                                                   fileId,
+                                                                                   excludedTypes == null ? new ArrayList<>() : excludedTypes,
+                                                                                   includeUnprocessed));
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java

@@ -276,11 +276,7 @@ public class ManualRedactionController implements ManualRedactionResource {
         accessControlService.verifyFileIsNotApproved(dossierId, fileId);
         accessControlService.verifyUserIsMemberOrApprover(dossierId);
 
-        List<ManualAddResponse> responseList = manualRedactionService.addRecategorization(dossierId,
-                                                                                          fileId,
-                                                                                          dossier,
-                                                                                          recategorizationRequests,
-                                                                                          includeUnprocessed);
+        List<ManualAddResponse> responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, includeUnprocessed);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReportTemplateController.java

@@ -109,6 +109,7 @@ public class ReportTemplateController implements ReportTemplateResource {
                         .multiFileReport(multiFileReport)
                         .build();
                 var reportTemplate = reportTemplateService.uploadTemplate(reportTemplateUploadRequest);
+                reportTemplatePlaceholderClient.evictReportTemplateCache();
                 auditPersistenceService.audit(AuditRequest.builder()
                                                       .userId(KeycloakSecurity.getUserId())
                                                       .objectId(reportTemplate.getTemplateId())
@@ -173,6 +174,7 @@ public class ReportTemplateController implements ReportTemplateResource {
         var storageId = reportTemplatePersistenceService.find(templateId).getStorageId();
         storageService.deleteObject(TenantContext.getTenantId(), storageId);
         reportTemplatePersistenceService.delete(templateId);
+        reportTemplatePlaceholderClient.evictReportTemplateCache();
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(templateId)

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -1,5 +1,6 @@
 package com.iqser.red.persistence.service.v1.external.api.impl.controller;
 
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.IMPORT_FILES;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.USE_SUPPORT_CONTROLLER;
 
 import java.io.IOException;
@@ -23,6 +24,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
@@ -132,7 +135,8 @@ public class SupportController implements SupportResource {
 
 
     @Override
-    public void importFiles(MultipartFile file) {
+    @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
+    public ImportResponse importFiles(MultipartFile file) {
 
         byte[] bytes;
         try {
@@ -140,7 +144,7 @@ public class SupportController implements SupportResource {
         } catch (IOException e) {
             throw new BadRequestException("File could not be read and is likely corrupted.", e);
         }
-        fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
+        return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
     }
 
 }

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java

@@ -388,7 +388,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
             return Collections.emptyList();
         }
 
-        List<ComponentDefinition> componentDefinitions = componentDefinitionService.createComponents(componentDefinitionAddRequests);
+        List<ComponentDefinition> componentDefinitions = componentDefinitionService.createComponents(dossierTemplateId, componentDefinitionAddRequests);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierTemplateId)

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DownloadControllerV2.java

@@ -27,34 +27,37 @@ public class DownloadControllerV2 implements DownloadResource {
     private final DownloadController downloadController;
     private final DownloadStatusPersistenceService downloadStatusPersistenceService;
 
+
     @Transactional
     public DownloadStatusList getDownloadStatusList() {
 
         var downloads = downloadStatusPersistenceService.getStatusesByUser(KeycloakSecurity.getUserId());
 
-        return new DownloadStatusList(downloads.stream().map(
-                status ->
-                        DownloadStatus.builder()
-                                .id(status.getUuid()) // This is a workaround the real id is the storageId.
-                                .userId(status.getUserId())
-                                .filename(status.getFilename())
-                                .mimeType(status.getMimeType())
-                                .errorCause(status.getErrorCause())
-                                .status(status.getStatus())
-                                .creationDate(status.getCreationDate())
-                                .lastDownload(status.getLastDownload())
-                                .fileSize(status.getFileSize())
-                                .dossierId(status.getDossier() != null ? status.getDossier().getId() : null)
-                                .fileIds(status.getFiles()
-                                                 .stream()
-                                                 .map(FileEntity::getId)
-                                                 .toList())
-                                .downloadFileTypes(status.getDownloadFileTypes()
-                                                           .stream()
-                                                           .toList())
-                                .reportTemplateIds(status.getReports().stream().map(ReportTemplateEntity::getTemplateId).toList())
-                                .build()).toList()
-        );
+        return new DownloadStatusList(downloads.stream()
+                                              .map(status -> DownloadStatus.builder()
+                                                      .id(status.getUuid()) // This is a workaround the real id is the storageId.
+                                                      .userId(status.getUserId())
+                                                      .filename(status.getFilename())
+                                                      .mimeType(status.getMimeType())
+                                                      .errorCause(status.getErrorCause())
+                                                      .status(status.getStatus())
+                                                      .creationDate(status.getCreationDate())
+                                                      .lastDownload(status.getLastDownload())
+                                                      .fileSize(status.getFileSize())
+                                                      .dossierId(status.getDossier() != null ? status.getDossier().getId() : null)
+                                                      .fileIds(status.getFiles()
+                                                                       .stream()
+                                                                       .map(FileEntity::getId)
+                                                                       .toList())
+                                                      .downloadFileTypes(status.getDownloadFileTypes()
+                                                                                 .stream()
+                                                                                 .toList())
+                                                      .reportTemplateIds(status.getReports()
+                                                                                 .stream()
+                                                                                 .map(ReportTemplateEntity::getTemplateId)
+                                                                                 .toList())
+                                                      .build())
+                                              .toList());
     }
 
 
@@ -81,7 +84,10 @@ public class DownloadControllerV2 implements DownloadResource {
                 .downloadFileTypes(status.getDownloadFileTypes()
                                            .stream()
                                            .toList())
-                .reportTemplateIds(status.getReports().stream().map(ReportTemplateEntity::getTemplateId).toList())
+                .reportTemplateIds(status.getReports()
+                                           .stream()
+                                           .map(ReportTemplateEntity::getTemplateId)
+                                           .toList())
                 .build();
     }
 

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java

@@ -178,6 +178,8 @@ public class ExternalControllerAdviceV2 {
         String message = Objects.requireNonNull(exception.getRootCause()).getMessage();
         if (message.contains("uq_component_definition_technical_name_template")) {
             message = "A component with the same technical name already exists in the given dossier template.";
+        } else if (message.contains("value too long for type character varying")) {
+            message = "Value is too long. Please use a shorter value.";
         } else {
             message = "Database error occurred.";
         }

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/FileControllerV2.java

@@ -176,10 +176,7 @@ public class FileControllerV2 implements FileResource {
 
         var storageId = downloadController.prepareDownload(PrepareDownloadWithOptionRequest.builder()
                                                                    .dossierId(dossierId)
-                                                                   .fileIds(statusController.getDossierStatus(dossierId)
-                                                                                    .stream()
-                                                                                    .map(FileStatus::getId)
-                                                                                    .toList())
+                                                                   .fileIds(bulkDownloadRequest.getFileIds())
                                                                    .reportTemplateIds(bulkDownloadRequest.getReportTemplateIds())
                                                                    .downloadFileTypes(bulkDownloadRequest.getDownloadFileTypes())
                                                                    .redactionPreviewColor(bulkDownloadRequest.getRedactionPreviewColor())

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMapper.java

@@ -34,11 +34,13 @@ public interface ComponentMapper {
 
     List<ComponentLogEntryValue> toComponentLogEntries(List<ComponentValue> values);
 
+
     Component toComponent(ComponentLogEntry entry);
 
 
     List<Component> toComponents(List<ComponentLogEntry> entries);
 
+
     ComponentLogEntry toComponentLogEntry(Component component);
 
 

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ComponentLogResource.java

@@ -49,9 +49,7 @@ public interface ComponentLogResource {
     @PostMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Adds overrides for components", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void addOverride(@PathVariable(DOSSIER_ID) String dossierId,
-                     @PathVariable(FILE_ID) String fileId,
-                     @RequestBody ComponentsOverrides componentsOverrides);
+    void addOverride(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody ComponentsOverrides componentsOverrides);
 
 
     @ResponseBody
@@ -59,8 +57,7 @@ public interface ComponentLogResource {
     @GetMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Gets overrides for components", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found")})
-    ComponentsOverrides getOverrides(@PathVariable(DOSSIER_ID) String dossierId,
-                                     @PathVariable(FILE_ID) String fileId);
+    ComponentsOverrides getOverrides(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId);
 
 
     @ResponseBody
@@ -68,8 +65,6 @@ public interface ComponentLogResource {
     @PostMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + "/revert" + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Reverts overrides for components", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void revertOverrides(@PathVariable(DOSSIER_ID) String dossierId,
-                         @PathVariable(FILE_ID) String fileId,
-                         @RequestBody RevertOverrideRequest revertOverrideRequest);
+    void revertOverrides(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody RevertOverrideRequest revertOverrideRequest);
 
 }

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/EntityLogResource.java

@@ -55,7 +55,9 @@ public interface EntityLogResource {
     @GetMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + "/pages", produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Gets the entity log for a fileId with all entities found on the given page numbers", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The dossier / file / entity log is not found.")})
-    EntityLogResponse getEntityLogWithEntriesOnPages(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestParam(value = "pageNumbers", defaultValue = "") List<Integer> pageNumbers);
+    EntityLogResponse getEntityLogWithEntriesOnPages(@PathVariable(DOSSIER_ID) String dossierId,
+                                                     @PathVariable(FILE_ID) String fileId,
+                                                     @RequestParam(value = "pageNumbers", defaultValue = "") List<Integer> pageNumbers);
 
 
     @GetMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + ANALYSIS_NUMBER_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -13,13 +13,15 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.multipart.MultipartFile;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
 
 import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 
@@ -54,15 +56,15 @@ public interface SupportResource {
     @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
     @Operation(summary = "Reanalyze all files in dossier template", description = """
             ## Reanalyze Files Endpoint
-            
+                        
             Use this endpoint to reanalyze all files in a specified Dossier Template. The reanalysis process can be tailored using various filtering options provided in the request body.
-            
+                        
             ### Parameters
-            
+                        
             - **DossierTemplateId**: Specifies the Dossier Template whose files need to be reanalyzed.
-            
+                        
             ### Request Body Configuration Options
-            
+                        
             - **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected for reanalysis.
             - **fileIds**: List of file IDs to filter. If empty, all files are selected for reanalysis.
             - **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated.
@@ -118,16 +120,16 @@ public interface SupportResource {
                          + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
             ## Export Files Endpoint
-            
+                        
             Use this endpoint to export a full Dossier Template, including all configurations, dossiers, and files.
             The endpoint returns a String storageId, which is used to query the DownloadController for the export zip archive's status and to download the archive.
-            
+                        
             ### Parameters
-            
+                        
             - **DossierTemplateId**: Specifies the Dossier Template to be exported.
-            
+                        
             ### Request Body Configuration Options
-            
+                        
             - **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected.
             - **fileIds**: List of file IDs to filter. If empty, all files are selected.
             - **excludeLayoutFiles**: Boolean. If true, excludes DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files.
@@ -139,11 +141,11 @@ public interface SupportResource {
     DownloadResponse exportFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody FileExchangeExportRequest exportRequest);
 
 
-    @ResponseStatus(value = HttpStatus.NO_CONTENT)
     @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files.")
+    @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. Returns the resulting dossierTemplateId.")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
-    void importFiles(@RequestPart(name = "file") MultipartFile file);
+    ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/BulkDownloadRequest.java

@@ -22,4 +22,5 @@ public class BulkDownloadRequest {
     private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
     private String redactionPreviewColor;
     private List<String> fileIds = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentOverrideList.java

@@ -19,4 +19,5 @@ public class ComponentOverrideList {
     String fileId;
 
     List<Component> componentOverrides = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/DossierAttributeDefinitionList.java

@@ -14,5 +14,6 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class DossierAttributeDefinitionList {
 
-    private List<DossierAttributeDefinition> dossierAttributeDefinitions= new ArrayList<>();
+    private List<DossierAttributeDefinition> dossierAttributeDefinitions = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/DossierStatusDefinitionList.java

@@ -15,4 +15,5 @@ import lombok.NoArgsConstructor;
 public class DossierStatusDefinitionList {
 
     private List<DossierStatusDefinition> dossierStatusDefinitions = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierResource.java

@@ -89,8 +89,8 @@ public interface DossierResource {
     @Operation(summary = "Update or set attributes for a specific dossier.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Dossier attributes successfully updated."), @ApiResponse(responseCode = "404", description = "Not found")})
     void setDossierAttributes(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of a dossier template", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
-                                                 @Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of a dossier", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
-                                                 @RequestBody DossierAttributes dossierAttributes);
+                              @Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of a dossier", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                              @RequestBody DossierAttributes dossierAttributes);
 
 
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml

@@ -415,7 +415,7 @@ paths:
     post:
       operationId: uploadMapping
       summary: Upload a new component mapping to a DossierTemplate.
-      description: |  
+      description: |
         Use this endpoint to upload a new component mapping to a specific DossierTemplate.
         
         #### File Requirements
@@ -434,7 +434,7 @@ paths:
         - The component mapping file can be utilized in component rules to relate components to existing master data.
         
         #### Example
-          
+        
           | search_value | mapped_value |
           |--------------|--------------|
           | Alice        | Manager      |
@@ -595,9 +595,9 @@ paths:
         - Component Definitions
       summary: Create new component definitions
       description: |
-       Create new component definitions for a given dossier template. The component will have a technical name which is automatically converted to snake case
-       that can't be updated after the creation. The rank is used to determine the order in which the components are displayed. The component's rank will
-       automatically be appended at the end based on the current number of components of this dossier template.
+        Create new component definitions for a given dossier template. The component will have a technical name which is automatically converted to snake case
+        that can't be updated after the creation. The rank is used to determine the order in which the components are displayed. The component's rank will
+        automatically be appended at the end based on the current number of components of this dossier template.
       parameters:
         - $ref: '#/components/parameters/dossierTemplateId'
       requestBody:
@@ -890,7 +890,7 @@ paths:
       summary: Retrieve a list of dossiers based on a specific dossier template.
       description: |
         By default, this endpoint provides a list of all *active* dossiers that are based on a specific dossier template.
-  
+        
         It returns an object containing an array of dossier objects under the key `dossiers`. Each dossier object contains 
         an identifier, a name, a description, an owner, and other fields with further details about the dossier.
         
@@ -2184,7 +2184,7 @@ components:
         A toggle to determine the deletion mode for a dossier:
         - `true`: The dossier will be permanently deleted and can't be restored.
         - `false` (default): Soft-delete, allowing restoration within the application-configured retention period.
-      
+        
         Using this parameter, you can also permanently delete a previously soft-deleted dossier during its retention period.
         
         > **Note:** Deleting a dossier also deletes all associated files.
@@ -2202,7 +2202,7 @@ components:
         - `true`: The file will be permanently deleted and can't be restored.
         - `false` (default): Soft-delete, allowing restoration within the retention period, provided its parent dossier
           hasn't been deleted. If the parent dossier has been deleted meanwhile, it needs to be restored first.
-    
+        
         Using this parameter, you can also permanently delete a previously soft-deleted file during its retention period.
     permanentlyDeleteFiles:
       name: deletePermanently
@@ -3324,7 +3324,7 @@ components:
           - a0044ae9-ddca-4f97-b0a1-3cb2517dbf39
           - c2e33246-e50a-4c43-831c-6789a5637db8
           - 6123fa16-6943-4b74-8524-54b0046a0ce6
-        approverIds: []
+        approverIds: [ ]
         reportTemplateIds:
           - daadea5f-917b-482a-b7d2-e65afe8f80ca
           - 8130acf6-4910-4123-827c-caacd8111402
@@ -3477,15 +3477,11 @@ components:
         The ComponentDefinitionAddRequest object represents a request to create a component definition within a dossier template.
         The rank will be automatically generated and does not need to be provided at creation.
       example:
-        dossierTemplateId: 8cd4b482-fb49-4315-9b51-789b4ae46c57
         technicalName: study_conclusion
         displayName: Study conclusion
         description: The conclusion of the study
       type: object
       properties:
-        dossierTemplateId:
-          description: The ID of the dossier template to which this component belongs.
-          type: string
         technicalName:
           description: The technical name of the component.
           type: string
@@ -3496,11 +3492,10 @@ components:
           description: A brief description of the component.
           type: string
       required:
-        - dossierTemplateId
         - technicalName
         - displayName
     ComponentDefinitionUpdateRequest:
-      description: | 
+      description: |
         The ComponentDefinitionUpdateRequest object represents a request to update a component definition within a dossier template. 
         Only the display name and description can be updated.
       example:
@@ -4632,10 +4627,10 @@ components:
         authorizationCode:
           authorizationUrl: /auth/realms/{workspaceId}/protocol/openid-connect/auth
           tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token
-          scopes: {}
+          scopes: { }
         password:
           tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token
-          scopes: {}
+          scopes: { }
         clientCredentials:
           tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token
-          scopes: {}
+          scopes: { }

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml

@@ -31,7 +31,7 @@ servers:
   - url: https://app.redactmanager.com
     description: RedactManager Cloud Service
 security:
-  - FF-OAUTH: []
+  - FF-OAUTH: [ ]
 tags:
   - name: 1. Dossier Templates
     description: Operations related to dossier templates.
@@ -258,7 +258,7 @@ paths:
       summary: Retrieve a list of dossiers based on a specific dossier template.
       description: |
         By default, this endpoint provides a list of all *active* dossiers that are based on a specific dossier template.
-  
+        
         It returns an object containing an array of dossier objects under the key `dossiers`. Each dossier object contains 
         an identifier, a name, a description, an owner, and other fields with further details about the dossier.
         
@@ -1309,7 +1309,7 @@ components:
         A toggle to determine the deletion mode for a dossier:
         - `true`: The dossier will be permanently deleted and can't be restored.
         - `false` (default): Soft-delete, allowing restoration within the application-configured retention period.
-      
+        
         Using this parameter, you can also permanently delete a previously soft-deleted dossier during its retention period.
         
         > **Note:** Deleting a dossier also deletes all associated files.
@@ -1327,7 +1327,7 @@ components:
         - `true`: The file will be permanently deleted and can't be restored.
         - `false` (default): Soft-delete, allowing restoration within the retention period, provided its parent dossier
           hasn't been deleted. If the parent dossier has been deleted meanwhile, it needs to be restored first.
-    
+        
         Using this parameter, you can also permanently delete a previously soft-deleted file during its retention period.
     permanentlyDeleteFiles:
       name: deletePermanently
@@ -3171,10 +3171,10 @@ components:
         authorizationCode:
           authorizationUrl: /auth/realms/{workspaceId}/protocol/openid-connect/auth
           tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token
-          scopes: {}
+          scopes: { }
         password:
           tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token
-          scopes: {}
+          scopes: { }
         clientCredentials:
           tokenUrl: /auth/realms/{workspaceId}/protocol/openid-connect/token
-          scopes: {}
+          scopes: { }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/dossier/DossierACLService.java

@@ -8,6 +8,7 @@ import java.util.stream.Collectors;
 import org.springframework.security.access.prepost.PostFilter;
 import org.springframework.security.acls.domain.ObjectIdentityImpl;
 import org.springframework.security.acls.domain.PrincipalSid;
+import org.springframework.security.acls.model.MutableAcl;
 import org.springframework.security.acls.model.MutableAclService;
 import org.springframework.security.acls.model.NotFoundException;
 import org.springframework.stereotype.Service;
@@ -110,7 +111,7 @@ public class DossierACLService extends AbstractACLService<String> {
     @Transactional
     public void updateDossierACL(Set<String> members, Set<String> approvers, String owner, String dossierId) {
 
-        var acl = getOrCreateACL(dossierId);
+        MutableAcl acl = getOrCreateACL(dossierId);
 
         // naive clear all ace's first
         while (!acl.getEntries().isEmpty()) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/dossier/ViewObjectDossierObjectPermissionService.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Set;
 import java.util.stream.Collectors;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/client/redactionreportservice/RssReportClient.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.persistence.management.v1.processor.client.redactionreportservice;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.redaction.report.v1.api.resource.RSSResource;
+
+@FeignClient(name = "RssReportClient", url = "${redaction-report-service.url}")
+public interface RssReportClient extends RSSResource {
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java

@@ -6,6 +6,7 @@ import static com.knecon.fforesight.service.layoutparser.internal.api.queue.Layo
 
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -193,6 +194,7 @@ public class MessagingConfiguration {
 
 
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrStatusUpdateResponseQueue() {
 
         return QueueBuilder.durable(OCR_STATUS_UPDATE_RESPONSE_QUEUE)
@@ -203,6 +205,7 @@ public class MessagingConfiguration {
 
 
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrStatusUpdateResponseDQL() {
 
         return QueueBuilder.durable(OCR_STATUS_UPDATE_RESPONSE_DQL).build();
@@ -239,6 +242,7 @@ public class MessagingConfiguration {
 
 
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrRequestQueue() {
 
         return QueueBuilder.durable(OCR_REQUEST_QUEUE)
@@ -249,19 +253,20 @@ public class MessagingConfiguration {
                 .build();
     }
 
+
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrResponseQueue() {
 
-        return QueueBuilder.durable(OCR_RESPONSE_QUEUE)
-                .build();
+        return QueueBuilder.durable(OCR_RESPONSE_QUEUE).build();
     }
 
 
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrDLQ() {
 
-        return QueueBuilder.durable(OCR_DLQ)
-                .build();
+        return QueueBuilder.durable(OCR_DLQ).build();
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/WebSocketSecurityConfig.java

@@ -73,6 +73,7 @@ public class WebSocketSecurityConfig extends AbstractSecurityWebSocketMessageBro
 
     }
 
+
     private Optional<String> extractTenantId(Message<?> message) {
 
         StompHeaderAccessor sha = StompHeaderAccessor.wrap(message);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierExportService.java

@@ -0,0 +1,70 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.nio.file.Path;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DossierExportService {
+
+    DictionaryPersistenceService dictionaryPersistenceService;
+    FileExportService fileExportService;
+    FileStatusManagementService fileStatusManagementService;
+    EntityTypeExportService entityTypeExportService;
+    ObjectMapper mapper;
+
+
+    @SneakyThrows
+    @Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
+    public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
+
+        List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
+
+        if (!request.dossierIds().contains(dossier.getId()) //
+            && files.stream()
+                    .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
+            // dossier has no files in requested files and dossier not explicitly requested -> don't export it.
+            return;
+        }
+
+        Path dossierFolder = folder.resolve(dossier.getId());
+        archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
+
+        for (FileModel fileEntity : files) {
+            if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
+                continue;
+            }
+            fileExportService.addFileToArchive(archiver, dossierFolder, request, fileEntity);
+        }
+
+        List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
+
+        String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
+
+        for (TypeEntity type : types) {
+            entityTypeExportService.addEntityTypeToArchive(archiver, type, entitiesFolder);
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java

@@ -0,0 +1,85 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DossierImportService {
+
+    DossierCreatorService dossierCreatorService;
+    FileImportService fileImportService;
+    EntityTypeImportService entityTypeImportService;
+
+
+    @Observed(name = "DossierImportService", contextualName = "import-dossier")
+    public void importDossier(FileExchangeImportModel fileExchangeImportModel,
+                              String userId,
+                              Dossier dossierToImport,
+                              TemplateImportInfo templateImportInfo,
+                              List<String> allReportTemplateIds) {
+
+        String dossierId = getDossierForImport(templateImportInfo, dossierToImport, userId, allReportTemplateIds);
+
+        entityTypeImportService.importEntityTypes(templateImportInfo.getDossierTemplateId(),
+                                                  dossierId,
+                                                  fileExchangeImportModel.getDossierTypes()
+                                                          .get(dossierToImport.getId()));
+
+        List<FileExchangeImportModel.FileImport> files = fileExchangeImportModel.getFiles().getOrDefault(dossierToImport.getId(), Collections.emptyList());
+
+        fileImportService.importFilesInParallel(files, userId, templateImportInfo, dossierId);
+    }
+
+
+    private String getDossierForImport(TemplateImportInfo templateImportInfo, Dossier dossier, String userId, List<String> reportTemplateIds) {
+
+        String dossierTemplateId = templateImportInfo.getDossierTemplateId();
+
+        CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
+                .dossierName(dossier.getName())
+                .description(dossier.getDescription())
+                .downloadFileTypes(dossier.getDownloadFileTypes())
+                .dueDate(dossier.getDueDate())
+                .dossierTemplateId(dossierTemplateId)
+                .requestingUser(userId)
+                .reportTemplateIds(reportTemplateIds)
+                .build();
+
+        Dossier importedDossier = null;
+        int retries = 0;
+        while (importedDossier == null && retries < 10) {
+            try {
+                importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
+            } catch (ConflictException e) {
+                retries++;
+                request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
+            }
+        }
+        if (importedDossier == null) {
+            throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
+        }
+        return importedDossier.getId();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java

@@ -100,18 +100,17 @@ public class DossierTemplateExportService {
     RabbitTemplate rabbitTemplate;
 
 
-    public JSONPrimitive<String> prepareExportDownload(ExportDownloadRequest request) {
+    public String prepareExportDownload(ExportDownloadRequest request) {
 
         var mimeType = "application/zip";
 
-        dossierTemplatePersistenceService.validateDossierTemplateForDuplicateRanks(request.getDossierTemplateId());
         String downloadFilename = request.getDossierTemplateId() + ".zip";
         String storageId = StorageIdUtils.getStorageId(request.getUserId(), request.getDossierTemplateId());
 
         downloadStatusPersistenceService.createStatus(request.getUserId(), storageId, downloadFilename, mimeType);
         addToExportDownloadQueue(ExportDownloadMessage.builder().dossierTemplateId(request.getDossierTemplateId()).storageId(storageId).userId(request.getUserId()).build(), 1);
 
-        return new JSONPrimitive<>(storageId);
+        return storageId;
     }
 
 
@@ -123,6 +122,7 @@ public class DossierTemplateExportService {
         });
     }
 
+
     @Observed(name = "DossierTemplateExportService", contextualName = "export-dossier-template")
     public long createDownloadArchive(ExportDownloadMessage downloadJob) throws IOException {
 
@@ -142,6 +142,7 @@ public class DossierTemplateExportService {
 
     }
 
+
     @Observed(name = "DossierTemplateExportService", contextualName = "write-dossier-template-to-archive")
     public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) throws IOException {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateImportService.java

@@ -21,6 +21,11 @@ import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ComponentMappingImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DossierTemplateArchiveReader;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
@@ -29,15 +34,12 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierStatusEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ComponentMappingImportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DossierTemplateArchiveReader;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReportTemplateService;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
@@ -56,17 +58,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkMo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ReportTemplateUploadRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
 import com.iqser.red.service.redaction.v1.model.DroolsValidation;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -83,6 +89,7 @@ public class DossierTemplateImportService {
     LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
     RulesPersistenceService rulesPersistenceService;
     DossierTemplatePersistenceService dossierTemplatePersistenceService;
+    DossierTemplateManagementService dossierTemplateManagementService;
     DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
     FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
     ColorsService colorsService;
@@ -121,10 +128,12 @@ public class DossierTemplateImportService {
     }
 
 
+    @Observed(name = "DossierTemplateImportService", contextualName = "import-template")
     public TemplateImportInfo importDossierTemplate(ImportTemplateResult request) {
 
         long start = System.currentTimeMillis();
-        String dossierTemplateId;
+        String dossierTemplateId = null;
+
         var dossierTemplateMeta = request.getDossierTemplate();
 
         TemplateImportInfo templateImportInfo = TemplateImportInfo.builder().build();
@@ -136,221 +145,277 @@ public class DossierTemplateImportService {
         }
 
         if (existingDossierTemplate != null) {
-            dossierTemplateId = existingDossierTemplate.getId();
+            dossierTemplateId = updateExistingDossierTemplate(request, existingDossierTemplate, dossierTemplateMeta, templateImportInfo);
+            templateImportInfo.setDossierTemplateId(dossierTemplateId);
 
-            // override the existing dossier template
-            updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId());
-            dossierTemplateRepository.save(existingDossierTemplate);
+            long elapsedTime = System.currentTimeMillis() - start;
+            log.info("Finished update of existing dossier template in {} ms for: {}", elapsedTime, dossierTemplateId);
+            return templateImportInfo;
+        }
 
-            existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(existingDossierTemplate)
-                                                                                                   .name()));
+        try {
+            importNewDossierTemplate(request, dossierTemplateMeta, templateImportInfo);
+            long elapsedTime = System.currentTimeMillis() - start;
+            log.info("Finished import dossier template in {} ms for: {}", elapsedTime, templateImportInfo.getDossierTemplateId());
 
-            // set colors
-            this.setColors(dossierTemplateId, request.getColors());
-
-            // set watermarks
-            if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
-                Set<String> toSetWatermarks = request.getWatermarks()
-                        .stream()
-                        .map(WatermarkModel::getName)
-                        .filter(Objects::nonNull)
-                        .collect(Collectors.toSet());
-                var currentWatermarkConfigs = watermarkService.getWatermarksForDossierTemplateId(dossierTemplateId);
-                Set<WatermarkEntity> configsToRemove = currentWatermarkConfigs.stream()
-                        .filter(c -> !toSetWatermarks.contains(c.getName()))
-                        .collect(Collectors.toSet());
-                var watermarkNameToEntity = currentWatermarkConfigs.stream()
-                        .collect(Collectors.toMap(WatermarkEntity::getName, Function.identity()));
-
-                request.getWatermarks()
-                        .forEach(watermark -> {
-                            log.info("watermark to add: " + watermark.getName());
-                            Long initialId = watermark.getId();
-                            if (!watermarkNameToEntity.isEmpty() && watermarkNameToEntity.get(watermark.getName()) != null) {
-
-                                watermark.setId(watermarkNameToEntity.get(watermark.getName()).getId());
-                            } else {
-                                watermark.setId(null);
-                            }
-                            watermark.setDossierTemplateId(dossierTemplateId);
-                            var entity = watermarkService.createOrUpdateWatermark(watermark);
-                            templateImportInfo.getLongMapping().put(initialId, entity.getId());
-                        });
-                configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId()));
+        } catch (Exception e) {
+            log.error("Error with message: {} occurred during import, undo import", e.getMessage(), e);
+            if (templateImportInfo != null && templateImportInfo.getDossierTemplateId() != null) {
+                dossierTemplateManagementService.deleteDossierTemplateCompletely(templateImportInfo.getDossierTemplateId());
             }
+            throw e;
+        }
 
-            // dossier status
-            if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
-                this.updateDossierStates(request, dossierTemplateId);
-            } else { // no states to add, delete current states
-                List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
-                currentStates.forEach(state -> dossierStatusPersistenceService.deleteDossierStatus(state.getId(), null));
-            }
+        return templateImportInfo;
+    }
 
-            // update dossier attributes
-            if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
-                this.updateDossierAttributes(request, dossierTemplateId, templateImportInfo);
-            } else { // no dossier attributes to add, but delete existing ones
-                var currentConfigs = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
-                currentConfigs.forEach(da -> dossierAttributeConfigPersistenceService.deleteDossierAttribute(da.getId()));
-            }
 
-            //update file attributes
-            if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
-                this.updateFileAttributes(request, dossierTemplateId, templateImportInfo);
-            } else { // no file attributes to add, but delete existing
-                var currentConfigs = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
-                currentConfigs.forEach(fa -> fileAttributeConfigPersistenceService.deleteFileAttribute(fa.getId()));
-            }
+    private void importNewDossierTemplate(ImportTemplateResult request, DossierTemplate dossierTemplateMeta, TemplateImportInfo templateImportInfo) {
 
-            entityTypeImportService.updateTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
+        String dossierTemplateId;
+        // creates new dossier template
+        if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
+            throw new ConflictException("DossierTemplate name must be set");
+        }
+        this.validateDossierTemplateName(dossierTemplateMeta);
+        DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
 
-            if (CollectionUtils.isNotEmpty(request.getComponentDefinitions())) {
-                this.updateComponents(request, dossierTemplateId);
-            } else { // no components to add, but remove existing ones
-                List<ComponentDefinitionEntity> currentComponents = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
-                this.deleteComponents(currentComponents);
-            }
+        // order is important
+        BeanUtils.copyProperties(dossierTemplateMeta, dossierTemplateEntity);
+        dossierTemplateEntity.setId(UUID.randomUUID().toString());
+        dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+        dossierTemplateEntity.setCreatedBy(request.getUserId());
+        //set rules
 
-            //set report templates
-            var existingReports = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplateId);
+        var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity);
+        loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate));
+        dossierTemplateId = loadedDossierTemplate.getId();
+        templateImportInfo.setDossierTemplateId(dossierTemplateId);
+        // set colors
+        this.updateColors(dossierTemplateId, request.getColors());
 
-            List<String> reportsUpdated = new ArrayList<>();
-            if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
-                request.getReportTemplateUploadRequests()
-                        .forEach(reportRequest -> {
-                            reportRequest.setDossierTemplateId(dossierTemplateId);
-                            var report = reportTemplateService.uploadTemplate(reportRequest);
-                            reportsUpdated.add(report.getTemplateId());
-                        });
-            }
-
-            // delete the reports that were not in the import
-            existingReports.forEach(r -> {
-                String storageId = r.getStorageId();
-                if (!reportsUpdated.contains(r.getTemplateId())) {
-                    storageService.deleteObject(TenantContext.getTenantId(), storageId);
-                    reportTemplatePersistenceService.delete(r.getTemplateId());
-                }
-            });
-
-            // set legal basis
-            if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
-                legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
-            } else { // delete existing
-                legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId);
-            }
-
-        } else {
-            // creates new dossier template
-            if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
-                throw new ConflictException("DossierTemplate name must be set");
-            }
-            this.validateDossierTemplateName(dossierTemplateMeta);
-            DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
-
-            // order is important
-            BeanUtils.copyProperties(dossierTemplateMeta, dossierTemplateEntity);
-            dossierTemplateEntity.setId(UUID.randomUUID().toString());
-            dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-            dossierTemplateEntity.setCreatedBy(request.getUserId());
-            //set rules
-
-            var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity);
-            loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate));
-
-            dossierTemplateId = loadedDossierTemplate.getId();
-            // set colors
-            this.setColors(dossierTemplateId, request.getColors());
-
-            // set watermarks
-            if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
-                request.getWatermarks()
-                        .forEach(watermark -> {
-                            Long initialId = watermark.getId();
-                            watermark.setDossierTemplateId(dossierTemplateId);
-                            watermark.setId(null);
-                            var en = watermarkService.createOrUpdateWatermark(watermark);
-                            templateImportInfo.getLongMapping().put(initialId, en.getId());
-                        });
-            }
-
-            // dossier status
-            if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
-                request.getDossierStatusInfos()
-                        .forEach(state -> {
-                            state.setId(null);
-                            this.updateDossierStatus(dossierTemplateId, state);
-                        });
-            }
-
-            //set dossier attributes
-            if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
-                request.getDossierAttributesConfigs()
-                        .forEach(da -> {
-                            String initialId = da.getId();
-                            da.setId(null); // in order to create a new dossier attribute
-                            var e = dossierAttributeConfigPersistenceService.addOrUpdateDossierAttribute(dossierTemplateId, convert(da, DossierAttributeConfigEntity.class));
-                            templateImportInfo.getIdMapping().put(initialId, e.getId());
-                        });
-            }
-
-            //set file attributes
-            if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
-                request.getFileAttributesConfigs()
-                        .forEach(fa -> {
-                            String initialId = fa.getId();
-                            fa.setId(null); // in order to force creation of new file attribute
-                            var e = fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId, convert(fa, FileAttributeConfigEntity.class));
-                            templateImportInfo.getIdMapping().put(initialId, e.getId());
-                        });
-            }
-
-            //set types
-            entityTypeImportService.importEntityTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
-
-            // set components
-            if (CollectionUtils.isNotEmpty(request.getComponentDefinitions())) {
-                for (ComponentDefinition componentDefinition : request.getComponentDefinitions()) {
-                    ComponentDefinitionAddRequest componentDefinitionAddRequest = ComponentDefinitionAddRequest.builder()
-                            .dossierTemplateId(dossierTemplateId)
-                            .displayName(componentDefinition.getDisplayName())
-                            .description(componentDefinition.getDescription())
-                            .technicalName(componentDefinition.getTechnicalName())
-                            .build();
-                    componentDefinitionPersistenceService.insert(componentDefinitionAddRequest, componentDefinition.getRank());
-                }
-            }
-
-            //set report templates
-            if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
-                request.getReportTemplateUploadRequests()
-                        .forEach(reportRequest -> {
-                            reportRequest.setDossierTemplateId(dossierTemplateId);
-                            reportTemplateService.uploadTemplate(reportRequest);
-                        });
-            }
-            // set legal basis
-            if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
-                legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
+        // set watermarks
+        if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
+            for (WatermarkModel watermark : request.getWatermarks()) {
+                Long initialId = watermark.getId();
+                watermark.setDossierTemplateId(dossierTemplateId);
+                watermark.setId(null);
+                var en = watermarkService.createOrUpdateWatermark(watermark);
+                templateImportInfo.getLongMapping().put(initialId, en.getId());
             }
         }
 
+        // dossier status
+        if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
+            for (DossierStatusInfo state : request.getDossierStatusInfos()) {
+                state.setId(null);
+                this.updateDossierStatus(dossierTemplateId, state);
+            }
+        }
+
+        //set dossier attributes
+        if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
+            for (DossierAttributeConfig da : request.getDossierAttributesConfigs()) {
+                String initialId = da.getId();
+                da.setId(null); // in order to create a new dossier attribute
+                var e = dossierAttributeConfigPersistenceService.addOrUpdateDossierAttribute(dossierTemplateId, convert(da, DossierAttributeConfigEntity.class));
+                templateImportInfo.getIdMapping().put(initialId, e.getId());
+            }
+        }
+
+        //set file attributes
+        if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
+            for (FileAttributeConfig fa : request.getFileAttributesConfigs()) {
+                String initialId = fa.getId();
+                fa.setId(null); // in order to force creation of new file attribute
+                var e = fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId, convert(fa, FileAttributeConfigEntity.class));
+                templateImportInfo.getIdMapping().put(initialId, e.getId());
+            }
+        }
+
+        //set types
+        entityTypeImportService.importEntityTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
+
+        // set components
+        if (CollectionUtils.isNotEmpty(request.getComponentDefinitions())) {
+            for (ComponentDefinition componentDefinition : request.getComponentDefinitions()) {
+                ComponentDefinitionAddRequest componentDefinitionAddRequest = ComponentDefinitionAddRequest.builder()
+                        .displayName(componentDefinition.getDisplayName())
+                        .description(componentDefinition.getDescription())
+                        .technicalName(componentDefinition.getTechnicalName())
+                        .build();
+                componentDefinitionPersistenceService.insert(componentDefinitionAddRequest, componentDefinition.getRank(), dossierTemplateId);
+            }
+        }
+
+        //set report templates
+        if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
+            for (ReportTemplateUploadRequest reportRequest : request.getReportTemplateUploadRequests()) {
+                reportRequest.setDossierTemplateId(dossierTemplateId);
+                reportTemplateService.uploadTemplate(reportRequest);
+            }
+        }
+        // set legal basis
+        if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
+            legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
+        }
+        setRulesWhenCompiled(request, dossierTemplateId);
+        setComponentMappings(dossierTemplateId, request.getComponentMappings());
+        setFileAttributeGeneralConfig(request, dossierTemplateId);
+
+    }
+
+
+    private void setFileAttributeGeneralConfig(ImportTemplateResult request, String dossierTemplateId) {
         // file attribute general configuration
         if (request.getFileAttributesGeneralConfiguration() != null) {
             fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
                                                                                  convert(request.getFileAttributesGeneralConfiguration(),
                                                                                          FileAttributesGeneralConfigurationEntity.class));
         }
+    }
 
+
+    private String updateExistingDossierTemplate(ImportTemplateResult request,
+                                                 DossierTemplateEntity existingDossierTemplate,
+                                                 DossierTemplate dossierTemplateMeta,
+                                                 TemplateImportInfo templateImportInfo) {
+
+        String dossierTemplateId;
+        dossierTemplateId = existingDossierTemplate.getId();
+        updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId());
+        dossierTemplateRepository.save(existingDossierTemplate);
+        existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(existingDossierTemplate)
+                                                                                               .name()));
+        updateColors(dossierTemplateId, request.getColors());
+        updateWatermarks(request, templateImportInfo, dossierTemplateId);
+        updateDossiers(request, dossierTemplateId);
+        updateDossierAttributes(request, templateImportInfo, dossierTemplateId);
+        updateFileAttributes(request, templateImportInfo, dossierTemplateId);
+        entityTypeImportService.updateTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
+        updateComponents(request, dossierTemplateId);
+        var existingReports = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplateId);
+        updateReports(request, dossierTemplateId, existingReports);
+        updateLegalBasis(request, dossierTemplateId);
         setRulesWhenCompiled(request, dossierTemplateId);
         setComponentMappings(dossierTemplateId, request.getComponentMappings());
+        setFileAttributeGeneralConfig(request, dossierTemplateId);
+        return dossierTemplateId;
+    }
 
-        templateImportInfo.setDossierTemplateId(dossierTemplateId);
 
-        long elapsedTime = System.currentTimeMillis() - start;
-        log.info("Finished import dossier template : {} for: {}", elapsedTime, dossierTemplateId);
-        return templateImportInfo;
+    private void updateLegalBasis(ImportTemplateResult request, String dossierTemplateId) {
+        // set legal basis
+        if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
+            legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
+        } else { // delete existing
+            legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId);
+        }
+    }
+
+
+    private void updateReports(ImportTemplateResult request, String dossierTemplateId, List<ReportTemplateEntity> existingReports) {
+
+        List<String> reportsUpdated = new ArrayList<>();
+        if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
+            for (ReportTemplateUploadRequest reportRequest : request.getReportTemplateUploadRequests()) {
+                reportRequest.setDossierTemplateId(dossierTemplateId);
+                var report = reportTemplateService.uploadTemplate(reportRequest);
+                reportsUpdated.add(report.getTemplateId());
+            }
+        }
+
+        // delete the reports that were not in the import
+        existingReports.forEach(r -> {
+            String storageId = r.getStorageId();
+            if (!reportsUpdated.contains(r.getTemplateId())) {
+                storageService.deleteObject(TenantContext.getTenantId(), storageId);
+                reportTemplatePersistenceService.delete(r.getTemplateId());
+            }
+        });
+    }
+
+
+    private void updateComponents(ImportTemplateResult request, String dossierTemplateId) {
+
+        if (CollectionUtils.isNotEmpty(request.getComponentDefinitions())) {
+            request.getComponentDefinitions()
+                    .forEach(componentDefinition -> {
+                        componentDefinition.setDossierTemplateId(dossierTemplateId);
+                        componentDefinitionPersistenceService.insert(MagicConverter.convert(componentDefinition, ComponentDefinitionEntity.class));
+                    });
+        } else { // no components to add, but remove existing ones
+            List<ComponentDefinitionEntity> currentComponents = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
+            currentComponents.forEach(componentDefinition -> {
+                componentDefinitionPersistenceService.delete(componentDefinition.getId());
+            });
+        }
+    }
+
+
+    private void updateFileAttributes(ImportTemplateResult request, TemplateImportInfo templateImportInfo, String dossierTemplateId) {
+        //update file attributes
+        if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
+            this.updateFileAttributes(request, dossierTemplateId, templateImportInfo);
+        } else { // no file attributes to add, but delete existing
+            var currentConfigs = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
+            currentConfigs.forEach(fa -> fileAttributeConfigPersistenceService.deleteFileAttribute(fa.getId()));
+        }
+    }
+
+
+    private void updateWatermarks(ImportTemplateResult request, TemplateImportInfo templateImportInfo, String dossierTemplateId) {
+        // set watermarks
+        if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
+            Set<String> toSetWatermarks = request.getWatermarks()
+                    .stream()
+                    .map(WatermarkModel::getName)
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toSet());
+            var currentWatermarkConfigs = watermarkService.getWatermarksForDossierTemplateId(dossierTemplateId);
+            Set<WatermarkEntity> configsToRemove = currentWatermarkConfigs.stream()
+                    .filter(c -> !toSetWatermarks.contains(c.getName()))
+                    .collect(Collectors.toSet());
+            var watermarkNameToEntity = currentWatermarkConfigs.stream()
+                    .collect(Collectors.toMap(WatermarkEntity::getName, Function.identity()));
+
+            for (WatermarkModel watermarkModel : request.getWatermarks()) {
+                log.info("watermark to add: " + watermarkModel.getName());
+                Long initialId = watermarkModel.getId();
+                if (!watermarkNameToEntity.isEmpty() && watermarkNameToEntity.get(watermarkModel.getName()) != null) {
+
+                    watermarkModel.setId(watermarkNameToEntity.get(watermarkModel.getName()).getId());
+                } else {
+                    watermarkModel.setId(null);
+                }
+                watermarkModel.setDossierTemplateId(dossierTemplateId);
+                var entity = watermarkService.createOrUpdateWatermark(watermarkModel);
+                templateImportInfo.getLongMapping().put(initialId, entity.getId());
+            }
+            configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId()));
+        }
+    }
+
+
+    private void updateDossiers(ImportTemplateResult request, String dossierTemplateId) {
+        // dossier status
+        if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
+            this.updateDossierStates(request, dossierTemplateId);
+        } else { // no states to add, delete current states
+            List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
+            currentStates.forEach(state -> dossierStatusPersistenceService.deleteDossierStatus(state.getId(), null));
+        }
+    }
+
+
+    private void updateDossierAttributes(ImportTemplateResult request, TemplateImportInfo templateImportInfo, String dossierTemplateId) {
+
+        if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
+            this.updateDossierAttributes(request, dossierTemplateId, templateImportInfo);
+        } else { // no dossier attributes to add, but delete existing ones
+            var currentConfigs = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
+            currentConfigs.forEach(da -> dossierAttributeConfigPersistenceService.deleteDossierAttribute(da.getId()));
+        }
     }
 
 
@@ -434,7 +499,7 @@ public class DossierTemplateImportService {
     }
 
 
-    private void setColors(String dossierTemplateId, Colors requestedColors) {
+    private void updateColors(String dossierTemplateId, Colors requestedColors) {
         // set colors
         if (requestedColors != null) {
             ColorsEntity colorsEntity = convert(requestedColors, ColorsEntity.class);
@@ -529,24 +594,6 @@ public class DossierTemplateImportService {
     }
 
 
-    private void updateComponents(ImportTemplateResult request, String dossierTemplateId) {
-
-        request.getComponentDefinitions()
-                .forEach(componentDefinition -> {
-                    componentDefinition.setDossierTemplateId(dossierTemplateId);
-                    componentDefinitionPersistenceService.insert(MagicConverter.convert(componentDefinition, ComponentDefinitionEntity.class));
-                });
-    }
-
-
-    private void deleteComponents(List<ComponentDefinitionEntity> componentDefinitionEntities) {
-
-        componentDefinitionEntities.forEach(componentDefinition -> {
-            componentDefinitionPersistenceService.delete(componentDefinition.getId());
-        });
-    }
-
-
     private void validateDossierTemplateName(DossierTemplate dossierTemplateMeta) {
 
         int nameSuffix = 0;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/EntityTypeExportService.java

@@ -41,6 +41,7 @@ public class EntityTypeExportService {
     ObjectMapper mapper;
     EntryPersistenceService entryPersistenceService;
 
+
     @Observed(name = "EntityTypeExportService", contextualName = "export-entity-type")
     public void addEntityTypeToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, TypeEntity typeEntity, String folder) throws JsonProcessingException {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/EntityTypeImportService.java

@@ -96,6 +96,9 @@ public class EntityTypeImportService {
     @Observed(name = "EntityTypeImportService", contextualName = "import-entity-type")
     public void importEntityTypes(String dossierTemplateId, String dossierId, EntityTypeImportModel importModel) {
 
+        if (importModel == null) {
+            return;
+        }
         enrichObservation(importModel);
 
         for (Type type : importModel.getTypes()) {
@@ -168,7 +171,7 @@ public class EntityTypeImportService {
     private void addEntries(Map<String, List<String>> entries, Map<String, List<String>> deleteEntries, String typeId, String typeName, DictionaryEntryType dictionaryType) {
 
         if (entries != null && !entries.isEmpty() && entries.get(typeName) != null && !entries.get(typeName).isEmpty()) {
-            dictionaryManagementService.addEntries(typeId, entries.get(typeName), true, true, dictionaryType);
+            dictionaryManagementService.addEntries(typeId, entries.get(typeName), true, true, dictionaryType, true);
         } else { // no entries, delete current entries
             List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryType, null)
                     .stream()

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeArchivalService.java

@@ -0,0 +1,38 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.FileExchangeArchiveReader;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
+import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FileExchangeArchivalService {
+
+    public static final long SIZE_THRESHOLD = 10000000000L; // 10GB
+    FileManagementServiceSettings settings;
+
+
+    @SneakyThrows
+    @Observed(name = "FileExchangeImportService", contextualName = "read-import-archive")
+    public FileExchangeImportModel readFileExchangeArchive(byte[] archive, String userId) {
+
+        FileExchangeArchiveReader fileExchangeArchiveReader = new FileExchangeArchiveReader(userId);
+        try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
+            zipEntryIterator.forEachRemaining(fileExchangeArchiveReader::handleZipEntryData);
+        }
+        return fileExchangeArchiveReader.buildResult();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java

@@ -3,28 +3,17 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
@@ -32,15 +21,11 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
-import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -53,16 +38,12 @@ public class FileExchangeExportService {
     DossierTemplateExportService dossierTemplateExportService;
     DossierTemplatePersistenceService dossierTemplatePersistenceService;
     DossierManagementService dossierManagementService;
-    FileStatusManagementService fileStatusManagementService;
     FileManagementStorageService storageService;
     DownloadStatusPersistenceService downloadStatusPersistenceService;
-    ManualChangesExportService manualChangesExportService;
-    DictionaryPersistenceService dictionaryPersistenceService;
-    ComponentLogService componentLogService;
-    ObjectMapper mapper;
+
     RabbitTemplate rabbitTemplate;
-    EntityTypeExportService entityTypeExportService;
     DossierIdFileIdRequestValidator requestValidator;
+    DossierExportService dossierExportService;
 
 
     @Observed(name = "FileExchangeExportService", contextualName = "prepare-export")
@@ -116,7 +97,7 @@ public class FileExchangeExportService {
                     continue;
                 }
 
-                addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
+                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
             }
 
             storeZipFile(downloadJob.getStorageId(), archiver);
@@ -126,125 +107,12 @@ public class FileExchangeExportService {
     }
 
 
-    @SneakyThrows
-    @Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
-    public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
-
-        List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
-
-        if (!request.dossierIds().contains(dossier.getId()) //
-            && files.stream()
-                    .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
-            // dossier has no files in requested files and dossier not explicitly requested -> don't export it.
-            return;
-        }
-
-        Path dossierFolder = folder.resolve(dossier.getId());
-        archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
-
-        for (FileModel fileEntity : files) {
-            if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
-                continue;
-            }
-            addFileToArchive(archiver, dossierFolder, request, fileEntity);
-        }
-
-        List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
-
-        String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
-
-        for (TypeEntity type : types) {
-            entityTypeExportService.addEntityTypeToArchive(archiver, type, entitiesFolder);
-        }
-    }
-
-
-    @SneakyThrows
-    @Observed(name = "FileExchangeExportService", contextualName = "export-file")
-    public void addFileToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, FileModel file) {
-
-        Path fileFolder = folder.resolve(file.getId());
-
-        if (!request.excludeAnalysisLogs()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
-                                                                        mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
-            if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
-                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                            buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
-                                                                            mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
-                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                            buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
-                                                                            mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
-            }
-        }
-
-        archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, FileExchangeNames.FILE_STATUS), mapper.writeValueAsBytes(file)));
-
-        if (!request.excludeLayoutFiles()) {
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.STRUCTURE);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TEXT);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
-        }
-
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.HIGHLIGHTS);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TABLES);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMAGES);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VISUAL_LAYOUT);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMPORTED);
-
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.ORIGIN);
-        if (!request.originFileOnly()) {
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.UNTOUCHED);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VIEWER);
-        }
-
-        if (!request.excludeFileAttributes()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.FILE_ATTRIBUTES),
-                                                                        mapper.writeValueAsBytes(file.getFileAttributes())));
-        }
-
-        if (!request.excludeManualRedactions()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.MANUAL_REDACTIONS),
-                                                                        mapper.writeValueAsBytes(manualChangesExportService.export(file.getId()))));
-        }
-    }
-
-
-    private static String buildFileName(FileModel file, String name) {
-
-        return buildFileName(file.getId(), name);
-    }
-
-
     public static String buildFileName(String fileId, String name) {
 
         return fileId + "." + name;
     }
 
 
-    @SneakyThrows
-    private void addArchiveModelForStorageFile(FileSystemBackedArchiver archiver, FileModel file, Path fileFolder, FileExchangeNames.Definition definition) {
-
-        String storageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), definition.fileType());
-        if (!storageService.objectExists(storageId)) {
-            log.debug("File {} not found in storage with id {}, skipping...", definition.fileType(), storageId);
-            return;
-        }
-        log.debug("Adding file {} from storage with id {} to archive.", definition.fileType(), storageId);
-
-        byte[] data = storageService.getObject(TenantContext.getTenantId(), storageId).readAllBytes();
-        FileSystemBackedArchiver.ArchiveModel archiveModel = new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, definition.getName()), data);
-        archiver.addEntry(archiveModel);
-    }
-
-
     @Observed(name = "FileExchangeExportService", contextualName = "store-archive")
     private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeImportService.java

@@ -1,37 +1,24 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
 
-import java.io.FileInputStream;
-import java.nio.file.Files;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
-import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.FileExchangeArchiveReader;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
-import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
-import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
-import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 
-import io.micrometer.common.KeyValue;
 import io.micrometer.observation.ObservationRegistry;
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -41,25 +28,20 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class FileExchangeImportService {
 
-    public static final long SIZE_THRESHOLD = 10000000000L; // 10GB
-
-    FileManagementServiceSettings settings;
     DossierTemplateImportService dossierTemplateImportService;
-    EntityTypeImportService entityTypeImportService;
-    DossierCreatorService dossierCreatorService;
-    FileImportService fileImportService;
-    FileManagementStorageService storageService;
-    ComponentLogService componentLogService;
+    DossierImportService dossierImportService;
+    FileExchangeArchivalService fileExchangeArchivalService;
     ReportTemplatePersistenceService reportTemplateService;
     ObservationRegistry registry;
+    DossierTemplateManagementService dossierTemplateManagementService;
 
 
     @Observed(name = "FileExchangeImportService", contextualName = "import-files")
-    public String importFileExchangeArchive(String userId, byte[] archive) {
+    public ImportResponse importFileExchangeArchive(String userId, byte[] archive) {
 
         long start = System.currentTimeMillis();
         log.info("Starting file import for user {}", userId);
-        FileExchangeImportModel fileExchangeImportModel = readFileExchangeArchive(archive, userId);
+        FileExchangeImportModel fileExchangeImportModel = fileExchangeArchivalService.readFileExchangeArchive(archive, userId);
         log.info("Parsed file import archive with {} dossiers and {} files",
                  fileExchangeImportModel.getDossiers().size(),
                  fileExchangeImportModel.getFiles().values()
@@ -70,7 +52,36 @@ public class FileExchangeImportService {
 
         enrichObservation(userId, archive, fileExchangeImportModel);
 
-        return dossierTemplateId;
+        return new ImportResponse(dossierTemplateId);
+    }
+
+
+    private String importFileExchangeModel(FileExchangeImportModel fileExchangeImportModel, String userId) {
+
+        ImportTemplateResult importTemplateResult = fileExchangeImportModel.getImportTemplateResult();
+
+        if (importTemplateResult == null) {
+            throw new BadRequestException("DossierTemplate not present in import archive.");
+        }
+
+        TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importTemplateResult);
+
+        List<String> allReportTemplateIds = reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
+                .stream()
+                .map(ReportTemplateEntity::getTemplateId)
+                .toList();
+        try {
+            for (Dossier dossierToImport : fileExchangeImportModel.getDossiers()) {
+                dossierImportService.importDossier(fileExchangeImportModel, userId, dossierToImport, templateImportInfo, allReportTemplateIds);
+            }
+            return templateImportInfo.getDossierTemplateId();
+        } catch (Exception e) {
+            log.error("Error with message {} ocurred, deleting imported dossier template again.", e.getMessage());
+            if (templateImportInfo.getDossierTemplateId() != null) {
+                dossierTemplateManagementService.deleteDossierTemplateCompletely(templateImportInfo.getDossierTemplateId());
+            }
+            throw e;
+        }
     }
 
 
@@ -88,108 +99,4 @@ public class FileExchangeImportService {
         }
     }
 
-
-    @Observed(name = "FileExchangeImportService", contextualName = "write-import-data-to-storage")
-    private String importFileExchangeModel(FileExchangeImportModel fileExchangeImportModel, String userId) {
-
-        TemplateImportInfo templateImportInfo = getDossierTemplateForImport(fileExchangeImportModel);
-
-        List<String> allReportTemplateIds = reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
-                .stream()
-                .map(ReportTemplateEntity::getTemplateId)
-                .toList();
-
-        for (Dossier dossierToImport : fileExchangeImportModel.getDossiers()) {
-            String dossierId = getDossierForImport(templateImportInfo, dossierToImport, userId, allReportTemplateIds);
-
-            entityTypeImportService.importEntityTypes(templateImportInfo.getDossierTemplateId(),
-                                                      dossierId,
-                                                      fileExchangeImportModel.getDossierTypes()
-                                                              .get(dossierToImport.getId()));
-
-            List<FileExchangeImportModel.FileImport> files = fileExchangeImportModel.getFiles().getOrDefault(dossierToImport.getId(), Collections.emptyList());
-
-            for (FileExchangeImportModel.FileImport file : files) {
-                // separate service for transactions
-                String fileId = fileImportService.saveFileToDb(userId, file, dossierId, templateImportInfo);
-
-                file.getFilesToUpload()
-                        .forEach(fileToUpload -> {
-                            try (FileInputStream in = new FileInputStream(fileToUpload.file().toFile())) {
-                                storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileToUpload.fileType()), in);
-                                Files.deleteIfExists(fileToUpload.file());
-                            } catch (Exception e) {
-                                throw new RuntimeException(e);
-                            }
-                        });
-
-                storageService.saveEntityLog(dossierId, fileId, file.getEntityLog());
-                if (file.getComponentLog() != null) {
-                    storageService.saveComponentLog(dossierId, fileId, file.getComponentLog());
-                }
-                if (file.getOverrides() != null) {
-                    file.getOverrides()
-                            .forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
-                }
-            }
-
-        }
-        return templateImportInfo.getDossierTemplateId();
-    }
-
-
-    private String getDossierForImport(TemplateImportInfo templateImportInfo, Dossier dossier, String userId, List<String> reportTemplateIds) {
-
-        String dossierTemplateId = templateImportInfo.getDossierTemplateId();
-
-        CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
-                .dossierName(dossier.getName())
-                .description(dossier.getDescription())
-                .downloadFileTypes(dossier.getDownloadFileTypes())
-                .dueDate(dossier.getDueDate())
-                .dossierTemplateId(dossierTemplateId)
-                .requestingUser(userId)
-                .reportTemplateIds(reportTemplateIds)
-                .build();
-
-        Dossier importedDossier = null;
-        int retries = 0;
-        while (importedDossier == null && retries < 10) {
-            try {
-                importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
-            } catch (ConflictException e) {
-                retries++;
-                request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
-            }
-        }
-        if (importedDossier == null) {
-            throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
-        }
-        return importedDossier.getId();
-    }
-
-
-    private TemplateImportInfo getDossierTemplateForImport(FileExchangeImportModel fileExchangeImportModel) {
-
-        ImportTemplateResult importTemplateResult = fileExchangeImportModel.getImportTemplateResult();
-
-        if (importTemplateResult == null) {
-            throw new BadRequestException("DossierTemplate not present in import archive.");
-        }
-
-        return dossierTemplateImportService.importDossierTemplate(importTemplateResult);
-    }
-
-
-    @SneakyThrows
-    @Observed(name = "FileExchangeImportService", contextualName = "read-import-archive")
-    private FileExchangeImportModel readFileExchangeArchive(byte[] archive, String userId) {
-
-        FileExchangeArchiveReader fileExchangeArchiveReader = new FileExchangeArchiveReader(userId);
-        try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
-            zipEntryIterator.forEachRemaining(fileExchangeArchiveReader::handleZipEntryData);
-        }
-        return fileExchangeArchiveReader.buildResult();
-    }
-
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExportService.java

@@ -0,0 +1,122 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.nio.file.Path;
+
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
+import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FileExportService {
+
+    ManualChangesExportService manualChangesExportService;
+    ComponentLogService componentLogService;
+    ObjectMapper mapper;
+    FileManagementStorageService storageService;
+
+
+    @SneakyThrows
+    @Observed(name = "FileExchangeExportService", contextualName = "export-file")
+    public void addFileToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, FileModel file) {
+
+        Path fileFolder = folder.resolve(file.getId());
+
+        if (!request.excludeAnalysisLogs()) {
+            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                        buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
+                                                                        mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
+            if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
+                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                            buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
+                                                                            mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
+                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                            buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
+                                                                            mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
+            }
+        }
+
+        archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, FileExchangeNames.FILE_STATUS), mapper.writeValueAsBytes(file)));
+
+        if (!request.excludeLayoutFiles()) {
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.STRUCTURE);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TEXT);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
+        }
+
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.HIGHLIGHTS);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TABLES);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMAGES);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VISUAL_LAYOUT);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMPORTED);
+
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.ORIGIN);
+        if (!request.originFileOnly()) {
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.UNTOUCHED);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VIEWER);
+        }
+
+        if (!request.excludeFileAttributes()) {
+            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                        buildFileName(file, FileExchangeNames.FILE_ATTRIBUTES),
+                                                                        mapper.writeValueAsBytes(file.getFileAttributes())));
+        }
+
+        if (!request.excludeManualRedactions()) {
+            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                        buildFileName(file, FileExchangeNames.MANUAL_REDACTIONS),
+                                                                        mapper.writeValueAsBytes(manualChangesExportService.export(file.getId()))));
+        }
+    }
+
+
+    private static String buildFileName(FileModel file, String name) {
+
+        return buildFileName(file.getId(), name);
+    }
+
+
+    public static String buildFileName(String fileId, String name) {
+
+        return fileId + "." + name;
+    }
+
+
+    @SneakyThrows
+    private void addArchiveModelForStorageFile(FileSystemBackedArchiver archiver, FileModel file, Path fileFolder, FileExchangeNames.Definition definition) {
+
+        String storageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), definition.fileType());
+        if (!storageService.objectExists(storageId)) {
+            log.debug("File {} not found in storage with id {}, skipping...", definition.fileType(), storageId);
+            return;
+        }
+        log.debug("Adding file {} from storage with id {} to archive.", definition.fileType(), storageId);
+
+        byte[] data = storageService.getObject(TenantContext.getTenantId(), storageId).readAllBytes();
+        FileSystemBackedArchiver.ArchiveModel archiveModel = new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, definition.getName()), data);
+        archiver.addEntry(archiveModel);
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileImportPersistenceService.java

@@ -0,0 +1,66 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileEntityMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
+
+import io.micrometer.observation.annotation.Observed;
+import jakarta.transaction.Transactional;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FileImportPersistenceService {
+
+    ManualChangesImportService manualChangesImportService;
+    FileRepository fileRepository;
+
+
+    @Transactional
+    @Observed(name = "FileImportPersistenceService", contextualName = "import-file-to-db")
+    public synchronized String saveFileToDb(String userId,
+                                            FileExchangeImportModel.FileImport file,
+                                            String dossierId,
+                                            TemplateImportInfo templateImportInfo) { // synchronized as this is being called in an async block. Might lock up DB otherwise.
+
+        String fileId = createFile(dossierId, userId, file, templateImportInfo.getIdMapping());
+
+        manualChangesImportService.importManualChanges(file.getManualChanges(), fileId, userId);
+
+        return fileId;
+    }
+
+
+    private String createFile(String dossierId, String userid, FileExchangeImportModel.FileImport file, Map<String, String> fileAttributeConfigMap) {
+
+        FileModel fileModel = file.getFileModel();
+
+        fileModel.setId(UploadService.generateFileId(fileModel.getFilename(), dossierId));
+
+        FileEntity fileEntity = MagicConverter.convert(fileModel, FileEntity.class, new FileEntityMapper(fileAttributeConfigMap));
+
+        fileEntity.setAssignee(userid);
+        fileEntity.setLastReviewer(userid);
+        fileEntity.setLastApprover(userid);
+        fileEntity.setUploader(userid);
+
+        fileEntity.setDossierId(dossierId);
+
+        fileRepository.save(fileEntity);
+
+        return fileEntity.getId();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileImportService.java

@@ -1,57 +1,80 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
 
-import java.util.Map;
+import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
-import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
-import com.iqser.red.service.persistence.management.v1.processor.utils.FileEntityMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
-import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
+import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
-import jakarta.transaction.Transactional;
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class FileImportService {
 
-
-    ManualChangesImportService manualChangesImportService;
-    FileRepository fileRepository;
+    FileImportPersistenceService fileImportPersistenceService;
+    FileManagementStorageService storageService;
+    ComponentLogService componentLogService;
 
 
-    @Transactional
-    public String saveFileToDb(String userId, FileExchangeImportModel.FileImport file, String dossierId, TemplateImportInfo templateImportInfo) {
+    @Observed(name = "FileImportService", contextualName = "import-files-in-parallel")
+    public void importFilesInParallel(List<FileExchangeImportModel.FileImport> files, String userId, TemplateImportInfo templateImportInfo, String dossierId) {
 
-        String fileId = createFile(dossierId, userId, file, templateImportInfo.getIdMapping());
+        List<CompletableFuture<Void>> fileImportFutures = new LinkedList<>();
 
-        manualChangesImportService.importManualChanges(file.getManualChanges(), fileId);
+        String tenantId = TenantContext.getTenantId();
 
-        return fileId;
+        for (FileExchangeImportModel.FileImport file : files) {
+            fileImportFutures.add(CompletableFuture.supplyAsync(() -> {
+                TenantContext.setTenantId(tenantId);
+                importFile(userId, templateImportInfo, file, dossierId);
+                return null;
+            }));
+        }
+        fileImportFutures.forEach(CompletableFuture::join);
     }
 
 
-    private String createFile(String dossierId, String userid, FileExchangeImportModel.FileImport file, Map<String, String> fileAttributeConfigMap) {
+    @Observed(name = "FileImportService", contextualName = "import-file")
+    public void importFile(String userId, TemplateImportInfo templateImportInfo, FileExchangeImportModel.FileImport file, String dossierId) {
 
-        FileModel fileModel = file.getFileModel();
+        // separate service for transactions
+        String fileId = fileImportPersistenceService.saveFileToDb(userId, file, dossierId, templateImportInfo);
 
-        fileModel.setId(UploadService.generateFileId(fileModel.getFilename(), dossierId));
-
-        FileEntity fileEntity = MagicConverter.convert(fileModel, FileEntity.class, new FileEntityMapper(fileAttributeConfigMap));
-
-        fileEntity.setDossierId(dossierId);
-
-        fileRepository.save(fileEntity);
-
-        return fileEntity.getId();
+        file.getFilesToUpload()
+                .forEach(fileToUpload -> {
+                    try (FileInputStream in = new FileInputStream(fileToUpload.file().toFile())) {
+                        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileToUpload.fileType()), in);
+                        Files.deleteIfExists(fileToUpload.file());
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+        if (file.getEntityLog() != null) {
+            storageService.saveEntityLog(dossierId, fileId, file.getEntityLog());
+        }
+        if (file.getComponentLog() != null) {
+            storageService.saveComponentLog(dossierId, fileId, file.getComponentLog());
+        }
+        if (file.getOverrides() != null) {
+            file.getOverrides()
+                    .forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
+        }
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/ManualChangesExportService.java

@@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
 
+import io.micrometer.observation.annotation.Observed;
 import jakarta.transaction.Transactional;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -33,33 +34,34 @@ public class ManualChangesExportService {
 
 
     @Transactional
+    @Observed(name = "ManualChangesExportService", contextualName = "export-manual-changes")
     public ManualChangesExportModel export(String fileId) {
 
         ManualChangesExportModel exportModel = new ManualChangesExportModel();
         exportModel.setManualAddExportModels(addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.all())
-                                               .stream()
-                                               .map(mapper::toModel)
-                                               .toList());
+                                                     .stream()
+                                                     .map(mapper::toModel)
+                                                     .toList());
         exportModel.setManualRemoveExportModels(removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.all())
-                                                  .stream()
-                                                  .map(mapper::toModel)
-                                                  .toList());
+                                                        .stream()
+                                                        .map(mapper::toModel)
+                                                        .toList());
         exportModel.setManualForceExportModels(forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.all())
-                                                 .stream()
-                                                 .map(mapper::toModel)
-                                                 .toList());
+                                                       .stream()
+                                                       .map(mapper::toModel)
+                                                       .toList());
         exportModel.setManualLegalBasisChangeExportModels(legalBasisChangePersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.all())
-                                                            .stream()
-                                                            .map(mapper::toModel)
-                                                            .toList());
+                                                                  .stream()
+                                                                  .map(mapper::toModel)
+                                                                  .toList());
         exportModel.setManualRecategorizationExportModels(recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.all())
-                                                            .stream()
-                                                            .map(mapper::toModel)
-                                                            .toList());
+                                                                  .stream()
+                                                                  .map(mapper::toModel)
+                                                                  .toList());
         exportModel.setManualResizeExportModels(resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.all())
-                                                  .stream()
-                                                  .map(mapper::toModel)
-                                                  .toList());
+                                                        .stream()
+                                                        .map(mapper::toModel)
+                                                        .toList());
         return exportModel;
 
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/ManualChangesImportService.java

@@ -13,6 +13,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository;
 
+import io.micrometer.observation.annotation.Observed;
 import jakarta.transaction.Transactional;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -36,38 +37,39 @@ public class ManualChangesImportService {
 
 
     @Transactional
-    public void importManualChanges(ManualChangesExportModel exportModel, String fileId) {
+    @Observed(name = "ManualChangesImportService", contextualName = "import-manual-changes")
+    public void importManualChanges(ManualChangesExportModel exportModel, String fileId, String userId) {
 
         FileEntity fileEntity = fileRepository.findById(fileId)
                 .orElseThrow();
         exportModel.getManualAddExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(manualRedactionRepository::saveAndFlush);
 
         exportModel.getManualForceExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(forceRedactionRepository::saveAndFlush);
 
         exportModel.getManualRecategorizationExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(recategorizationRepository::saveAndFlush);
 
         exportModel.getManualRemoveExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(removeRedactionRepository::saveAndFlush);
 
         exportModel.getManualResizeExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(resizeRedactionRepository::saveAndFlush);
 
         exportModel.getManualLegalBasisChangeExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(legalBasisChangeRepository::saveAndFlush);
 
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java

@@ -70,7 +70,7 @@ public class ZipEntryIterator implements Iterator<ZipEntryData>, AutoCloseable {
 
                 double compressionRatio = (double) totalSizeEntry / nextEntry.getCompressedSize();
                 if (compressionRatio > compressionThresholdRatio) {
-                    throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio );
+                    throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio);
                 }
             }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java

@@ -4,7 +4,6 @@ import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.hibernate.annotations.Fetch;
 import org.springframework.data.annotation.LastModifiedDate;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/AnnotationEntityId.java

@@ -4,7 +4,6 @@ import java.io.Serializable;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Embeddable;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/CommentEntity.java

@@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotat
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.GeneratedValue;
@@ -9,9 +11,6 @@ import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/RectangleEntity.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.annotations;
 
 import jakarta.persistence.Embeddable;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ViewedPageEntity.java

@@ -3,6 +3,8 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotat
 import java.io.Serializable;
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Embeddable;
 import jakarta.persistence.EmbeddedId;
@@ -11,9 +13,6 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.MapsId;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/audit/AuditEntity.java

@@ -4,6 +4,8 @@ import java.time.OffsetDateTime;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
+
 import jakarta.persistence.Basic;
 import jakarta.persistence.Column;
 import jakarta.persistence.Convert;
@@ -13,9 +15,6 @@ import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ApplicationConfigurationEntity.java

@@ -4,7 +4,6 @@ import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ColorsEntity.java

@@ -8,7 +8,6 @@ import jakarta.persistence.Id;
 import jakarta.persistence.PrePersist;
 import jakarta.persistence.PreUpdate;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java

@@ -6,7 +6,6 @@ import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java

@@ -6,7 +6,6 @@ import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DigitalSignatureEntity.java

@@ -1,13 +1,12 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.Lob;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DigitalSignatureKmsEntity.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
+
 import jakarta.persistence.Basic;
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
@@ -7,9 +9,6 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Lob;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DigitalSignatureTypeEntity.java

@@ -1,15 +1,14 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.DigitalSignatureType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
 import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.DigitalSignatureType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/FileAttributesGeneralConfigurationEntity.java

@@ -1,15 +1,14 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.MapsId;
 import jakarta.persistence.OneToOne;
 import jakarta.persistence.Table;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisEntity.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configu
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Embeddable;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisMappingEntity.java

@@ -3,11 +3,15 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configu
 import java.util.ArrayList;
 import java.util.List;
 
-import jakarta.persistence.*;
-
 import org.hibernate.annotations.Fetch;
 import org.hibernate.annotations.FetchMode;
 
+import jakarta.persistence.Column;
+import jakarta.persistence.ElementCollection;
+import jakarta.persistence.Entity;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java

@@ -2,6 +2,9 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configu
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
@@ -9,10 +12,6 @@ import jakarta.persistence.Id;
 import jakarta.persistence.JoinColumn;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/WatermarkEntity.java

@@ -2,6 +2,11 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configu
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.HorizontalTextAlignment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.VerticalTextAlignment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
@@ -13,12 +18,6 @@ import jakarta.persistence.Id;
 import jakarta.persistence.JoinColumn;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.HorizontalTextAlignment;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.VerticalTextAlignment;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/ComponentDefinitionEntity.java

@@ -4,8 +4,6 @@ import java.time.OffsetDateTime;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
 import jakarta.persistence.UniqueConstraint;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeConfigEntity.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
@@ -7,9 +9,6 @@ import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeEntity.java

@@ -7,7 +7,6 @@ import jakarta.persistence.Embeddable;
 import jakarta.persistence.EmbeddedId;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java

@@ -6,6 +6,17 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.hibernate.annotations.Fetch;
+import org.hibernate.annotations.FetchMode;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
+import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
+import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierVisibility;
+
 import jakarta.persistence.CascadeType;
 import jakarta.persistence.Column;
 import jakarta.persistence.Convert;
@@ -19,18 +30,6 @@ import jakarta.persistence.ManyToMany;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.Table;
-
-import org.hibernate.annotations.Fetch;
-import org.hibernate.annotations.FetchMode;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierVisibility;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierStatusEntity.java

@@ -3,6 +3,9 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier
 import java.util.ArrayList;
 import java.util.List;
 
+import org.hibernate.annotations.Fetch;
+import org.hibernate.annotations.FetchMode;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
@@ -11,10 +14,6 @@ import jakarta.persistence.JoinColumn;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.Table;
-
-import org.hibernate.annotations.Fetch;
-import org.hibernate.annotations.FetchMode;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeConfigEntity.java

@@ -1,5 +1,8 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
@@ -7,10 +10,6 @@ import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeEntity.java

@@ -7,7 +7,6 @@ import jakarta.persistence.Embeddable;
 import jakarta.persistence.EmbeddedId;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/ReportTemplateEntity.java

@@ -4,6 +4,8 @@ import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
@@ -11,9 +13,6 @@ import jakarta.persistence.Id;
 import jakarta.persistence.ManyToMany;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadRedactionFileStatusEntity.java

@@ -1,17 +1,21 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.download;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
 import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadRedactionFileDetailsConverter;
 
-import jakarta.persistence.*;
+import jakarta.persistence.Column;
+import jakarta.persistence.Convert;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.ArrayList;
-import java.util.List;
-
 @Data
 @Entity
 @Builder

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/index/IndexInformationEntity.java

@@ -2,13 +2,12 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.index;
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/MigrationEntity.java

@@ -6,7 +6,6 @@ import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/SaasMigrationStatusEntity.java

@@ -2,7 +2,12 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.migrati
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
 
-import jakarta.persistence.*;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.EnumType;
+import jakarta.persistence.Enumerated;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationEntity.java

@@ -4,6 +4,8 @@ import java.time.OffsetDateTime;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
+
 import jakarta.persistence.Basic;
 import jakarta.persistence.Column;
 import jakarta.persistence.Convert;
@@ -12,11 +14,7 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
-import jakarta.persistence.SequenceGenerator;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationPreferencesEntity.java

@@ -3,18 +3,17 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.notific
 import java.util.ArrayList;
 import java.util.List;
 
+import org.hibernate.annotations.Fetch;
+import org.hibernate.annotations.FetchMode;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.notification.EmailNotificationType;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.ElementCollection;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
-import org.hibernate.annotations.Fetch;
-import org.hibernate.annotations.FetchMode;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.notification.EmailNotificationType;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/permissions/CustomPermissionEntity.java

@@ -7,7 +7,6 @@ import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.IdClass;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/jobs/CreateJobsConfiguration.java

@@ -2,8 +2,6 @@ package com.iqser.red.service.persistence.management.v1.processor.jobs;
 
 import java.text.ParseException;
 
-import com.iqser.red.service.persistence.management.v1.processor.service.job.*;
-
 import org.quartz.CronExpression;
 import org.quartz.CronScheduleBuilder;
 import org.quartz.JobBuilder;
@@ -13,6 +11,15 @@ import org.quartz.TriggerBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import com.iqser.red.service.persistence.management.v1.processor.service.job.AnalysisFlagCalculationSchedulerJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.DeletedFilesCleanupJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadCleanupJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.KeyCloakUserSyncJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.SendNotificationEmailJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.SyncUserPermissionsJob;
+
 @Configuration
 public class CreateJobsConfiguration {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/LifecycleAspect.java

@@ -0,0 +1,57 @@
+package com.iqser.red.service.persistence.management.v1.processor.lifecycle;
+
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.springframework.amqp.AmqpRejectAndDontRequeueException;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Aspect
+@Component
+@Slf4j
+@RequiredArgsConstructor
+public class LifecycleAspect {
+
+    private final LifecycleManager lifecycleManager;
+
+    private final LifecycleProperties lifecycleProperties;
+
+
+    @Around("@annotation(org.springframework.amqp.rabbit.annotation.RabbitListener) || "
+            + "@annotation(org.springframework.web.bind.annotation.GetMapping) || "
+            + "@annotation(org.springframework.web.bind.annotation.PostMapping) || "
+            + "@annotation(org.springframework.web.bind.annotation.PutMapping) || "
+            + "@annotation(org.springframework.web.bind.annotation.DeleteMapping) || "
+            + "@annotation(org.springframework.web.bind.annotation.RequestMapping))")
+    public Object checkLifecycle(ProceedingJoinPoint joinPoint) throws Throwable {
+
+        String targetClassName = joinPoint.getTarget().getClass().getPackageName();
+        if (!targetClassName.startsWith(lifecycleProperties.getBasePackage())) {
+            return joinPoint.proceed();
+        }
+
+        synchronized (lifecycleManager) {
+            if (!lifecycleManager.isRunning()) {
+                log.info("Application is shutting down, rejecting new messages.");
+                throw new AmqpRejectAndDontRequeueException("Application is shutting down, rejecting new messages.");
+            }
+            lifecycleManager.incrementAndGet();
+        }
+
+        try {
+            return joinPoint.proceed();
+        } finally {
+            int remainingTasks = lifecycleManager.decrementAndGet();
+            synchronized (lifecycleManager) {
+                if (remainingTasks == 0 && !lifecycleManager.isRunning()) {
+                    lifecycleManager.countDown();
+                    log.info("All tasks are done, ready for shutdown.");
+                }
+            }
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/LifecycleManager.java

@@ -0,0 +1,106 @@
+package com.iqser.red.service.persistence.management.v1.processor.lifecycle;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.springframework.context.ApplicationListener;
+import org.springframework.context.SmartLifecycle;
+import org.springframework.context.event.ContextClosedEvent;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class LifecycleManager implements SmartLifecycle, ApplicationListener<ContextClosedEvent> {
+
+    private volatile boolean running; // by default initialized as false
+    private volatile boolean shutdownInitiated; // by default initialized as false
+    private final Object shutdownMonitor = new Object();
+    private final AtomicInteger activeTasks = new AtomicInteger(0);
+    private final CountDownLatch latch = new CountDownLatch(1);
+
+
+    public void countDown() {
+
+        latch.countDown();
+    }
+
+
+    public int incrementAndGet() {
+
+        return activeTasks.incrementAndGet();
+    }
+
+
+    public int decrementAndGet() {
+
+        return activeTasks.decrementAndGet();
+    }
+
+
+    @Override
+    public void start() {
+
+        synchronized (shutdownMonitor) {
+            running = true;
+        }
+    }
+
+
+    @Override
+    public void stop() {
+
+        synchronized (shutdownMonitor) {
+            running = false;
+            if (activeTasks.get() == 0) {
+                latch.countDown(); // No active tasks, release the latch immediately
+            }
+        }
+    }
+
+
+    @Override
+    public boolean isRunning() {
+
+        return running;
+    }
+
+
+    @Override
+    public boolean isAutoStartup() {
+
+        return true;
+    }
+
+
+    @Override
+    public int getPhase() {
+
+        return Integer.MAX_VALUE; // Start this component last and stop it first
+    }
+
+
+    @Override
+    public void onApplicationEvent(ContextClosedEvent event) {
+
+        synchronized (shutdownMonitor) {
+            if (shutdownInitiated) {
+                return; // Avoid multiple shutdown initiations
+            }
+            shutdownInitiated = true;
+        }
+
+        stop();
+        log.info("Context is closing, waiting for ongoing tasks to complete.");
+        try {
+            latch.await(); // Wait for the latch to count down to zero
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            log.error("Shutdown was interrupted", e);
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/LifecycleProperties.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.persistence.management.v1.processor.lifecycle;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@ConfigurationProperties("lifecycle")
+public class LifecycleProperties {
+
+    private String basePackage;
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ManualChangesExportMapper.java

@@ -8,6 +8,12 @@ import org.mapstruct.MappingTarget;
 import org.mapstruct.Mappings;
 import org.mapstruct.factory.Mappers;
 
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualAddExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualForceExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualLegalBasisChangeExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualRecategorizationExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualRemoveExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualResizeExportModel;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity;
@@ -16,12 +22,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualAddExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualForceExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualLegalBasisChangeExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualRecategorizationExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualRemoveExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualResizeExportModel;
 
 @Mapper
 public interface ManualChangesExportMapper {
@@ -53,79 +53,96 @@ public interface ManualChangesExportMapper {
     ManualResizeExportModel toModel(ManualResizeRedactionEntity entity);
 
 
-    ManualLegalBasisChangeEntity toEntity(ManualLegalBasisChangeExportModel model, @Context FileEntity file);
+    ManualLegalBasisChangeEntity toEntity(ManualLegalBasisChangeExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    ManualForceRedactionEntity toEntity(ManualForceExportModel model, @Context FileEntity file);
+    ManualForceRedactionEntity toEntity(ManualForceExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    ManualRedactionEntryEntity toEntity(ManualAddExportModel model, @Context FileEntity file);
+    ManualRedactionEntryEntity toEntity(ManualAddExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    IdRemovalEntity toEntity(ManualRemoveExportModel model, @Context FileEntity file);
+    IdRemovalEntity toEntity(ManualRemoveExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    ManualRecategorizationEntity toEntity(ManualRecategorizationExportModel model, @Context FileEntity file);
+    ManualRecategorizationEntity toEntity(ManualRecategorizationExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    ManualResizeRedactionEntity toEntity(ManualResizeExportModel model, @Context FileEntity file);
+    ManualResizeRedactionEntity toEntity(ManualResizeExportModel model, @Context FileEntity file, @Context String userId);
 
 
     @AfterMapping
     default void setFileStatusAndId(ManualLegalBasisChangeExportModel model,
                                     @MappingTarget ManualLegalBasisChangeEntity.ManualLegalBasisChangeEntityBuilder entity,
-                                    @Context FileEntity file) {
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
     default void setFileStatusAndId(ManualRecategorizationExportModel model,
                                     @MappingTarget ManualRecategorizationEntity.ManualRecategorizationEntityBuilder entity,
-                                    @Context FileEntity file) {
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualAddExportModel model, @MappingTarget ManualRedactionEntryEntity.ManualRedactionEntryEntityBuilder entity, @Context FileEntity file) {
+    default void setFileStatusAndId(ManualAddExportModel model,
+                                    @MappingTarget ManualRedactionEntryEntity.ManualRedactionEntryEntityBuilder entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualResizeExportModel model, @MappingTarget ManualResizeRedactionEntity.ManualResizeRedactionEntityBuilder entity, @Context FileEntity file) {
+    default void setFileStatusAndId(ManualResizeExportModel model,
+                                    @MappingTarget ManualResizeRedactionEntity.ManualResizeRedactionEntityBuilder entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualForceExportModel model, @MappingTarget ManualForceRedactionEntity.ManualForceRedactionEntityBuilder entity, @Context FileEntity file) {
+    default void setFileStatusAndId(ManualForceExportModel model,
+                                    @MappingTarget ManualForceRedactionEntity.ManualForceRedactionEntityBuilder entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity.IdRemovalEntityBuilder entity, @Context FileEntity file) {
+    default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity.IdRemovalEntityBuilder entity, @Context FileEntity file, @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/OldQueueCleanup.java

@@ -0,0 +1,40 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration;
+
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE;
+
+import org.springframework.amqp.core.AmqpAdmin;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
+
+import lombok.RequiredArgsConstructor;
+
+// This can not run with migration starter service, it needs to be an InitializingBean otherwise it would be executed after queue listeners are initialized
+
+@Component
+@RequiredArgsConstructor
+public class OldQueueCleanup implements InitializingBean {
+
+    private final AmqpAdmin amqpAdmin;
+    private final FileManagementServiceSettings settings;
+
+    public static final String OCR_QUEUE = "ocrQueue";
+    public static final String OCR_DLQ = "ocrDLQ";
+
+
+    @Override
+    public void afterPropertiesSet() {
+
+        // This should only run in post upgrade hook
+        if (settings.isMigrateOnly()) {
+            amqpAdmin.deleteQueue(OCR_QUEUE);
+            amqpAdmin.deleteQueue(OCR_DLQ);
+            amqpAdmin.deleteQueue(OCR_STATUS_UPDATE_RESPONSE_QUEUE);
+            amqpAdmin.deleteQueue(OCR_STATUS_UPDATE_RESPONSE_DQL);
+            amqpAdmin.deleteQueue("ocr_status_update_response_dql");
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java

@@ -49,9 +49,11 @@ public class StorageToMongoCopyService {
             log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId);
             Optional<EntityLog> entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId);
             if (entityLogFromStorage.isPresent()) {
-                log.info("File found, now  saving in mongodb");
-                fileManagementStorageService.saveEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLogFromStorage.get());
+                log.info("File found, now saving in mongodb");
+                fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId);
+                fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLogFromStorage.get());
                 log.info("Deleting old file from storage");
+                fileManagementStorageService.storeJSONObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG_BAK, entityLogFromStorage.get());
                 fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG);
             }
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/AddGraphicDictionaryType19.java

@@ -0,0 +1,75 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Setter
+@Service
+public class AddGraphicDictionaryType19 extends Migration {
+
+    @Autowired
+    private DossierTemplatePersistenceService dossierTemplatePersistenceService;
+
+    @Autowired
+    private DictionaryPersistenceService dictionaryPersistenceService;
+
+    @Autowired
+    private TypeRepository typeRepository;
+
+    private static final String NAME = "Migration to add experimental Graphic Type";
+    private static final long VERSION = 19;
+
+
+    public AddGraphicDictionaryType19() {
+
+        super(NAME, VERSION);
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        dossierTemplatePersistenceService.getAllDossierTemplates()
+                .forEach(dossierTemplateEntity -> {
+
+                    String typeId = toTypeId("graphic", dossierTemplateEntity.getId(), null);
+
+                    typeRepository.findById(typeId).ifPresentOrElse(type -> {
+                        dictionaryPersistenceService.setExperimentalTrue(typeId);
+                        log.info("Updated Graphic Type {} to experimental", typeId);
+                    }, () -> {
+                        dictionaryPersistenceService.addType("graphic",
+                                                             dossierTemplateEntity.getId(),
+                                                             "#ffbdfd",
+                                                             "#8df06c",
+                                                             "#c498fa",
+                                                             998,
+                                                             true,
+                                                             true,
+                                                             false,
+                                                             "Empty dictionary used to configure graphic colors.",
+                                                             false,
+                                                             "Graphic",
+                                                             null,
+                                                             true,
+                                                             false,
+                                                             false,
+                                                             false,
+                                                             true);
+                        log.info("Added Graphic Type {}", typeId);
+                    });
+                });
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentOverride.java

@@ -18,4 +18,5 @@ public class ComponentOverride {
 
     String name;
     List<ComponentLogEntryValue> componentOverrideValues = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ManualChangesQueryOptions.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.persistence.management.v1.processor.model;
 
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 
 import lombok.Builder;
 import lombok.Getter;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/AnalyseEvent.java

@@ -21,7 +21,9 @@ public class AnalyseEvent {
     private int numberOfPagesToOCR;
     private int numberOfOCRedPages;
 
+
     public AnalyseEvent(String dossierId, String fileId, AnalyseStatus analyseStatus, Integer analysisNumber, OffsetDateTime timestamp) {
+
         this.dossierId = dossierId;
         this.fileId = fileId;
         this.analyseStatus = analyseStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DossierEvent.java

@@ -10,6 +10,8 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 @AllArgsConstructor
 public class DossierEvent {
+
     private String dossierId;
     private DossierEventType dossierEventType;
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DossierEventType.java

@@ -1,5 +1,11 @@
 package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
 
 public enum DossierEventType {
-    CREATE, UPDATE, SOFT_DELETE, HARD_DELETE, UNDELETE, ARCHIVE, UNARCHIVE
+    CREATE,
+    UPDATE,
+    SOFT_DELETE,
+    HARD_DELETE,
+    UNDELETE,
+    ARCHIVE,
+    UNARCHIVE
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DownloadEvent.java

@@ -16,4 +16,5 @@ public class DownloadEvent {
     private String downloadId;
     private String userId;
     private DownloadStatusValue status;
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/FileEvent.java

@@ -15,5 +15,4 @@ public class FileEvent {
     private String fileId;
     private FileEventType fileEventType;
 
-
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/FileEventType.java

@@ -1,5 +1,9 @@
 package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
 
 public enum FileEventType {
-    CREATE, UPDATE, SOFT_DELETE, HARD_DELETE, UNDELETE
+    CREATE,
+    UPDATE,
+    SOFT_DELETE,
+    HARD_DELETE,
+    UNDELETE
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/NotificationEvent.java

@@ -12,4 +12,5 @@ import lombok.NoArgsConstructor;
 public class NotificationEvent {
 
     private String userId;
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java

@@ -174,8 +174,7 @@ public final class ActionRoles {
     public static final String DEPLOYMENT_INFO = "red-deployment-info";
 
     public static final String USE_SUPPORT_CONTROLLER = "red-use-support-controller";
-
-
+    public static final String IMPORT_FILES = "red-import-files";
 
 
     private ActionRoles() {}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ApplicationRoles.java

@@ -33,6 +33,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_REPORT_TEMPLATES;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_RSS;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_TENANTS;
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.IMPORT_FILES;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.MANAGE_ACL_PERMISSIONS;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.MANAGE_USER_PREFERENCES;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.MANAGE_VIEWED_PAGES;
@@ -116,12 +117,19 @@ public final class ApplicationRoles {
 
     public static final Set<String> UNMAPPED_ACTION_ROLES = Sets.newHashSet(UNARCHIVE_DOSSIER, UPDATE_LICENSE, GET_RSS, USE_SUPPORT_CONTROLLER);
 
-    public static final Set<String> KNECON_ADMIN_ACTION_ROLES = Sets.newHashSet(READ_LICENSE, UPDATE_LICENSE, GET_TENANTS, CREATE_TENANT, READ_USERS, READ_ALL_USERS, WRITE_USERS,
-                                                                                READ_SMTP_CONFIGURATION,
-                                                                                WRITE_SMTP_CONFIGURATION,
-                                                                                UNARCHIVE_DOSSIER,
-                                                                                USE_SUPPORT_CONTROLLER);
-    public static final Set<String> KNECON_SUPPORT_ACTION_ROLES = Sets.newHashSet(KNECON_ADMIN_ACTION_ROLES);
+    public static final Set<String> KNECON_SUPPORT_ACTION_ROLES = Sets.newHashSet(READ_LICENSE,
+                                                                                  UPDATE_LICENSE,
+                                                                                  GET_TENANTS,
+                                                                                  CREATE_TENANT,
+                                                                                  READ_USERS,
+                                                                                  READ_ALL_USERS,
+                                                                                  WRITE_USERS,
+                                                                                  READ_SMTP_CONFIGURATION,
+                                                                                  WRITE_SMTP_CONFIGURATION,
+                                                                                  UNARCHIVE_DOSSIER,
+                                                                                  USE_SUPPORT_CONTROLLER);
+
+    public static final Set<String> KNECON_ADMIN_ACTION_ROLES = Sets.union(KNECON_SUPPORT_ACTION_ROLES, Sets.newHashSet(IMPORT_FILES));
 
     public static final Set<String> RED_USER_ACTION_ROLES = Sets.newHashSet(ADD_COMMENT,
                                                                             READ_LICENSE,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java

@@ -117,7 +117,7 @@ public class AnalysisFlagsCalculationService {
                 hasUpdates = true;
             }
 
-            if (entry.getNumberOfComments() > 0) {
+            if (entry.getNumberOfComments() > 0 && !entry.getState().equals(EntryState.REMOVED)) {
                 hasComments = true;
             }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentDefinitionService.java

@@ -6,17 +6,21 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
 
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ComponentDefinitionEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ComponentDefinitionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionUpdateRequest;
@@ -32,28 +36,33 @@ import lombok.experimental.FieldDefaults;
 public class ComponentDefinitionService {
 
     ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
+    DossierTemplateRepository dossierTemplateRepository;
 
 
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DICTIONARY_TYPE + "')")
     @Transactional
-    public List<ComponentDefinition> createComponents(List<ComponentDefinitionAddRequest> componentDefinitionAddRequests) {
+    public List<ComponentDefinition> createComponents(String dossierTemplateId, List<ComponentDefinitionAddRequest> componentDefinitionAddRequests) {
+
+        validateDossierTemplateExists(dossierTemplateId);
+        validateAddRequest(componentDefinitionAddRequests);
 
-        validateComponentRequest(componentDefinitionAddRequests);
         List<ComponentDefinitionEntity> componentEntities = new ArrayList<>();
-        componentDefinitionAddRequests.forEach(componentDefinitionAddRequest -> componentEntities.add(componentDefinitionPersistenceService.insert(componentDefinitionAddRequest)));
+        componentDefinitionAddRequests.forEach(componentDefinitionAddRequest -> componentEntities.add(componentDefinitionPersistenceService.insert(componentDefinitionAddRequest,
+                                                                                                                                                   dossierTemplateId)));
         return componentEntities.stream()
                 .map(componentDefinitionEntity -> MagicConverter.convert(componentDefinitionEntity, ComponentDefinition.class))
+                .sorted(Comparator.comparing(ComponentDefinition::getRank))
                 .toList();
     }
 
 
-    private void validateComponentRequest(List<ComponentDefinitionAddRequest> componentDefinitionAddRequests) {
+    private void validateAddRequest(List<ComponentDefinitionAddRequest> componentDefinitionAddRequests) {
 
-        String firstDossierTemplateId = componentDefinitionAddRequests.get(0).getDossierTemplateId();
-        boolean allMatch = componentDefinitionAddRequests.stream()
-                .allMatch(request -> request.getDossierTemplateId().equals(firstDossierTemplateId));
-        if (!allMatch) {
-            throw new BadRequestException("All components must have the same dossierTemplateId.");
+        long nrOfComponentsWithEmptyTechnicalName = componentDefinitionAddRequests.stream()
+                .filter(c -> StringUtils.isEmpty(c.getTechnicalName()))
+                .count();
+        if (nrOfComponentsWithEmptyTechnicalName != 0) {
+            throw new BadRequestException("Technical name can not be empty for a component.");
         }
     }
 
@@ -61,6 +70,7 @@ public class ComponentDefinitionService {
     @PreAuthorize("hasAuthority('" + READ_DICTIONARY_TYPES + "')")
     public ComponentDefinition getComponentByDossierTemplateIdAndComponentId(String dossierTemplateId, String componentId) {
 
+        validateDossierTemplateExists(dossierTemplateId);
         return MagicConverter.convert(componentDefinitionPersistenceService.findComponentByDossierTemplateIdAndComponentId(dossierTemplateId, componentId),
                                       ComponentDefinition.class);
     }
@@ -69,17 +79,20 @@ public class ComponentDefinitionService {
     @PreAuthorize("hasAuthority('" + READ_DICTIONARY_TYPES + "')")
     public List<ComponentDefinition> getComponentsByDossierTemplateId(String dossierTemplateId, boolean includeSoftDeleted) {
 
-        List<ComponentDefinitionEntity> entities;
+        validateDossierTemplateExists(dossierTemplateId);
+        List<ComponentDefinitionEntity> componentDefinitionEntities;
 
         if (includeSoftDeleted) {
-            entities = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
+            componentDefinitionEntities = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
         } else {
-            entities = componentDefinitionPersistenceService.findComponentsByDossierTemplateIdExcludeSoftDeleted(dossierTemplateId);
+            componentDefinitionEntities = componentDefinitionPersistenceService.findComponentsByDossierTemplateIdExcludeSoftDeleted(dossierTemplateId);
         }
 
-        return entities.stream()
+        return componentDefinitionEntities.stream()
                 .map(entity -> MagicConverter.convert(entity, ComponentDefinition.class))
+                .sorted(Comparator.comparingInt(ComponentDefinition::getRank))
                 .collect(Collectors.toList());
+
     }
 
 
@@ -87,6 +100,15 @@ public class ComponentDefinitionService {
     @Transactional
     public List<ComponentDefinition> updateComponents(String dossierTemplateId, List<ComponentDefinitionUpdateRequest> componentDefinitionUpdateRequests) {
 
+        validateDossierTemplateExists(dossierTemplateId);
+
+        long nrOfComponentUpdateRequestsWithEmptyDisplayName = componentDefinitionUpdateRequests.stream()
+                .filter(componentDefinitionUpdateRequest -> StringUtils.isEmpty(componentDefinitionUpdateRequest.getDisplayName()))
+                .count();
+        if (nrOfComponentUpdateRequestsWithEmptyDisplayName != 0) {
+            throw new BadRequestException("Component display name can not be empty");
+        }
+
         List<ComponentDefinitionEntity> componentEntities = new ArrayList<>();
         componentDefinitionUpdateRequests.forEach(componentDefinitionUpdateRequest -> {
             ComponentDefinitionEntity componentDefinitionEntity = componentDefinitionPersistenceService.findComponent(dossierTemplateId, componentDefinitionUpdateRequest.getId());
@@ -96,6 +118,7 @@ public class ComponentDefinitionService {
         });
         return componentEntities.stream()
                 .map(componentDefinitionEntity -> MagicConverter.convert(componentDefinitionEntity, ComponentDefinition.class))
+                .sorted(Comparator.comparing(ComponentDefinition::getRank))
                 .toList();
     }
 
@@ -104,8 +127,8 @@ public class ComponentDefinitionService {
     @Transactional
     public List<ComponentDefinitionEntity> deleteComponents(String dossierTemplateId, List<String> componentIds) {
 
-        OffsetDateTime now = OffsetDateTime.now();
-        return componentDefinitionPersistenceService.softDeleteComponents(dossierTemplateId, componentIds, now);
+        validateDossierTemplateExists(dossierTemplateId);
+        return componentDefinitionPersistenceService.softDeleteComponents(dossierTemplateId, componentIds, OffsetDateTime.now());
     }
 
 
@@ -113,6 +136,7 @@ public class ComponentDefinitionService {
     @Transactional
     public List<ComponentDefinition> restoreComponents(String dossierTemplateId, List<String> componentIds) {
 
+        validateDossierTemplateExists(dossierTemplateId);
 
         List<ComponentDefinitionEntity> componentDefinitionEntities = componentDefinitionPersistenceService.restoreComponents(dossierTemplateId, componentIds);
         int rank = componentDefinitionPersistenceService.countByDossierTemplateId(dossierTemplateId);
@@ -122,20 +146,24 @@ public class ComponentDefinitionService {
         }
         return componentDefinitionEntities.stream()
                 .map(componentDefinitionEntity -> MagicConverter.convert(componentDefinitionEntity, ComponentDefinition.class))
+                .sorted(Comparator.comparing(ComponentDefinition::getRank))
                 .toList();
     }
 
 
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DICTIONARY_TYPE + "')")
-    @Transactional
     public List<ComponentDefinition> reorderComponents(String dossierTemplateId, List<String> componentIds) {
 
+        validateDossierTemplateExists(dossierTemplateId);
+
         List<ComponentDefinition> orderedComponents = new ArrayList<>();
         List<ComponentDefinitionEntity> existingComponents = componentDefinitionPersistenceService.findComponentsByDossierTemplateIdExcludeSoftDeleted(dossierTemplateId);
 
         Map<String, ComponentDefinitionEntity> componentMap = existingComponents.stream()
                 .collect(Collectors.toMap(ComponentDefinitionEntity::getId, component -> component));
 
+        List<ComponentDefinitionEntity> remainingComponents = new ArrayList<>(existingComponents);
+
         int rank = 1;
         for (String componentId : componentIds) {
             ComponentDefinitionEntity component = componentMap.get(componentId);
@@ -143,10 +171,28 @@ public class ComponentDefinitionService {
                 component.setRank(rank++);
                 componentDefinitionPersistenceService.update(component);
                 orderedComponents.add(MagicConverter.convert(component, ComponentDefinition.class));
+                remainingComponents.remove(component);
             }
         }
 
+        // Continue updating ranks for remaining components
+        for (ComponentDefinitionEntity component : remainingComponents) {
+            component.setRank(rank++);
+            componentDefinitionPersistenceService.update(component);
+            orderedComponents.add(MagicConverter.convert(component, ComponentDefinition.class));
+        }
+
+        orderedComponents.sort(Comparator.comparing(ComponentDefinition::getRank));
+
         return orderedComponents;
     }
 
+
+    private void validateDossierTemplateExists(String dossierTemplateId) {
+
+        if (!dossierTemplateRepository.existsByIdAndNotDeleted(dossierTemplateId)) {
+            throw new NotFoundException(String.format("DossierTemplate with Id %s not found.", dossierTemplateId));
+        }
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentLogService.java

@@ -2,7 +2,9 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -15,6 +17,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotAl
 import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ComponentDefinitionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
 import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
@@ -28,9 +31,11 @@ import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
 
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class ComponentLogService {
 
     private final FileManagementStorageService fileManagementStorageService;
@@ -38,11 +43,16 @@ public class ComponentLogService {
     private final AuditPersistenceService auditPersistenceService;
     private final ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
     private final UserService userService;
+    private final DossierRepository dossierRepository;
 
 
     public ComponentLog getComponentLog(String dossierId, String fileId, boolean includeOverrides) {
 
-        List<ComponentDefinitionEntity> orderedEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierId);
+        List<ComponentDefinitionEntity> orderedEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierRepository.findDossierTemplateId(dossierId))
+                .stream()
+                .sorted(Comparator.comparing(ComponentDefinitionEntity::getRank))
+                .toList();
+        orderedEntities.forEach(o -> log.info("Name: {} Rank: {}", o.getTechnicalName(), o.getRank()));
         List<String> orderedNames = orderedEntities.stream()
                 .map(ComponentDefinitionEntity::getTechnicalName)
                 .collect(Collectors.toList());
@@ -81,17 +91,22 @@ public class ComponentLogService {
 
         List<ComponentLogEntry> componentLogEntries = componentLog.getComponentLogEntries();
 
-        List<ComponentLogEntry> sortedLogEntries = new ArrayList<>();
-        List<ComponentLogEntry> nonOrderedLogEntries = new ArrayList<>();
-
+        Map<String, ComponentLogEntry> entryMap = new HashMap<>();
         for (ComponentLogEntry entry : componentLogEntries) {
-            if (orderedNames.contains(entry.getName())) {
-                sortedLogEntries.add(entry);
-            } else {
-                nonOrderedLogEntries.add(entry);
-            }
+            entryMap.put(entry.getName().toLowerCase(Locale.getDefault()), entry);
         }
 
+        List<ComponentLogEntry> sortedLogEntries = new ArrayList<>();
+
+        orderedNames.forEach(name -> {
+            ComponentLogEntry entry = entryMap.get(name);
+            if (entry != null) {
+                entryMap.remove(name);
+                sortedLogEntries.add(entry);
+            }
+        });
+
+        List<ComponentLogEntry> nonOrderedLogEntries = new ArrayList<>(entryMap.values());
         nonOrderedLogEntries.sort(new ComponentOrderComparator(orderedNames));
         sortedLogEntries.addAll(nonOrderedLogEntries);
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java

@@ -7,19 +7,17 @@ import java.time.OffsetDateTime;
 import java.util.List;
 
 import org.springframework.dao.DataAccessException;
-import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
 import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ComponentMappingRepository;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
-import jakarta.validation.ConstraintViolationException;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java

@@ -222,6 +222,13 @@ public class ComponentMappingService {
     }
 
 
+    public void deleteByDossierTemplateId(String dossierTemplateId) {
+
+        componentMappingPersistenceService.getByDossierTemplateId(dossierTemplateId)
+                .forEach(cm -> componentMappingPersistenceService.deleteById(dossierTemplateId, cm.getId()));
+    }
+
+
     private static class CSVSorter implements Comparator<String[]> {
 
         @Override

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java

@@ -33,6 +33,7 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.TypeMappe
 import com.iqser.red.service.persistence.management.v1.processor.validation.DictionaryValidator;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.TypeRankSummary;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 
 import jakarta.transaction.Transactional;
@@ -238,6 +239,13 @@ public class DictionaryManagementService {
     @Transactional
     public void addEntries(String typeId, List<String> entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType) {
 
+        addEntries(typeId, entries, removeCurrent, ignoreInvalidEntries, dictionaryEntryType, false);
+    }
+
+
+    @Transactional
+    public void addEntries(String typeId, List<String> entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType, boolean isImport) {
+
         checkForDossierTypeExistenceAndCreate(typeId);
 
         Set<String> cleanEntries = entries.stream()
@@ -258,7 +266,7 @@ public class DictionaryManagementService {
 
         // To check whether the type exists, type should not be added into database implicitly by addEntry.
         Type typeResult = MagicConverter.convert(dictionaryPersistenceService.getType(typeId), Type.class);
-        if (!typeResult.isHasDictionary()) {
+        if (!typeResult.isHasDictionary() && !isImport) {
             throw new BadRequestException("Entity type does not have a dictionary");
         }
 
@@ -397,6 +405,12 @@ public class DictionaryManagementService {
     }
 
 
+    public void hardDeleteByDossierTemplateId(String dossierTemplateId) {
+
+        dictionaryPersistenceService.hardDeleteTypesByDossierTemplateId(dossierTemplateId);
+    }
+
+
     public long getCurrentVersion(String typeId) {
 
         Type typeResult = MagicConverter.convert(dictionaryPersistenceService.getType(typeId), Type.class);
@@ -426,4 +440,10 @@ public class DictionaryManagementService {
 
     }
 
+
+    public List<TypeRankSummary> getTypeRankSummaryList(String dossierTemplateId) {
+
+        return dictionaryPersistenceService.getTypeRankSummaryList(dossierTemplateId);
+    }
+
 }