reformat

RED-8951: made it possible to import entities with dictionary but hasDictionary=false

See merge request redactmanager/persistence-service!620

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -8,7 +8,7 @@ plugins {
 
 val redactionServiceVersion by rootProject.extra { "4.290.0" }
 val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" }
-val redactionReportServiceVersion by rootProject.extra { "4.64.0" }
+val redactionReportServiceVersion by rootProject.extra { "4.73.6" }
 val searchServiceVersion by rootProject.extra { "2.90.0" }
 
 repositories {

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java

@@ -381,7 +381,6 @@ public class DossierController implements DossierResource {
             throw new AccessDeniedException("Can not delete dossier that is owned by a different user");
         }
 
-
         dossierManagementService.softDeleteDossier(dossierId);
 
         auditPersistenceService.audit(AuditRequest.builder()

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/EntityLogController.java

@@ -37,7 +37,10 @@ public class EntityLogController implements EntityLogResource {
 
         accessControlService.checkViewPermissionsToDossier(dossierId);
         accessControlService.validateFileResourceExistence(fileId);
-        return entityLogResponseMapper.toLogResponse(entityLogService.getEntityLog(dossierId, fileId, excludedTypes == null ? new ArrayList<>() : excludedTypes, includeUnprocessed));
+        return entityLogResponseMapper.toLogResponse(entityLogService.getEntityLog(dossierId,
+                                                                                   fileId,
+                                                                                   excludedTypes == null ? new ArrayList<>() : excludedTypes,
+                                                                                   includeUnprocessed));
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java

@@ -276,11 +276,7 @@ public class ManualRedactionController implements ManualRedactionResource {
         accessControlService.verifyFileIsNotApproved(dossierId, fileId);
         accessControlService.verifyUserIsMemberOrApprover(dossierId);
 
-        List<ManualAddResponse> responseList = manualRedactionService.addRecategorization(dossierId,
+        List<ManualAddResponse> responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, includeUnprocessed);
-                                                                                          fileId,
-                                                                                          dossier,
-                                                                                          recategorizationRequests,
-                                                                                          includeUnprocessed);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReportTemplateController.java

@@ -109,6 +109,7 @@ public class ReportTemplateController implements ReportTemplateResource {
                         .multiFileReport(multiFileReport)
                         .build();
                 var reportTemplate = reportTemplateService.uploadTemplate(reportTemplateUploadRequest);
+                reportTemplatePlaceholderClient.evictReportTemplateCache();
                 auditPersistenceService.audit(AuditRequest.builder()
                                                       .userId(KeycloakSecurity.getUserId())
                                                       .objectId(reportTemplate.getTemplateId())
@@ -173,6 +174,7 @@ public class ReportTemplateController implements ReportTemplateResource {
         var storageId = reportTemplatePersistenceService.find(templateId).getStorageId();
         storageService.deleteObject(TenantContext.getTenantId(), storageId);
         reportTemplatePersistenceService.delete(templateId);
+        reportTemplatePlaceholderClient.evictReportTemplateCache();
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(templateId)

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -1,5 +1,6 @@
 package com.iqser.red.persistence.service.v1.external.api.impl.controller;
 
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.IMPORT_FILES;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.USE_SUPPORT_CONTROLLER;
 
 import java.io.IOException;
@@ -23,6 +24,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
@@ -132,7 +135,8 @@ public class SupportController implements SupportResource {
 
 
     @Override
-    public void importFiles(MultipartFile file) {
+    @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
+    public ImportResponse importFiles(MultipartFile file) {
 
         byte[] bytes;
         try {
@@ -140,7 +144,7 @@ public class SupportController implements SupportResource {
         } catch (IOException e) {
             throw new BadRequestException("File could not be read and is likely corrupted.", e);
         }
-        fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
+        return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
     }
 
 }

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DownloadControllerV2.java

@@ -27,14 +27,14 @@ public class DownloadControllerV2 implements DownloadResource {
     private final DownloadController downloadController;
     private final DownloadStatusPersistenceService downloadStatusPersistenceService;
 
+
     @Transactional
     public DownloadStatusList getDownloadStatusList() {
 
         var downloads = downloadStatusPersistenceService.getStatusesByUser(KeycloakSecurity.getUserId());
 
-        return new DownloadStatusList(downloads.stream().map(
+        return new DownloadStatusList(downloads.stream()
-                status ->
+                                              .map(status -> DownloadStatus.builder()
-                        DownloadStatus.builder()
                                                       .id(status.getUuid()) // This is a workaround the real id is the storageId.
                                                       .userId(status.getUserId())
                                                       .filename(status.getFilename())
@@ -52,9 +52,12 @@ public class DownloadControllerV2 implements DownloadResource {
                                                       .downloadFileTypes(status.getDownloadFileTypes()
                                                                                  .stream()
                                                                                  .toList())
-                                .reportTemplateIds(status.getReports().stream().map(ReportTemplateEntity::getTemplateId).toList())
+                                                      .reportTemplateIds(status.getReports()
-                                .build()).toList()
+                                                                                 .stream()
-        );
+                                                                                 .map(ReportTemplateEntity::getTemplateId)
+                                                                                 .toList())
+                                                      .build())
+                                              .toList());
     }
 
 
@@ -81,7 +84,10 @@ public class DownloadControllerV2 implements DownloadResource {
                 .downloadFileTypes(status.getDownloadFileTypes()
                                            .stream()
                                            .toList())
-                .reportTemplateIds(status.getReports().stream().map(ReportTemplateEntity::getTemplateId).toList())
+                .reportTemplateIds(status.getReports()
+                                           .stream()
+                                           .map(ReportTemplateEntity::getTemplateId)
+                                           .toList())
                 .build();
     }
 

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ExternalControllerAdviceV2.java

@@ -178,6 +178,8 @@ public class ExternalControllerAdviceV2 {
         String message = Objects.requireNonNull(exception.getRootCause()).getMessage();
         if (message.contains("uq_component_definition_technical_name_template")) {
             message = "A component with the same technical name already exists in the given dossier template.";
+        } else if (message.contains("value too long for type character varying")) {
+            message = "Value is too long. Please use a shorter value.";
         } else {
             message = "Database error occurred.";
         }

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/FileControllerV2.java

@@ -176,10 +176,7 @@ public class FileControllerV2 implements FileResource {
 
         var storageId = downloadController.prepareDownload(PrepareDownloadWithOptionRequest.builder()
                                                                    .dossierId(dossierId)
-                                                                   .fileIds(statusController.getDossierStatus(dossierId)
+                                                                   .fileIds(bulkDownloadRequest.getFileIds())
-                                                                                    .stream()
-                                                                                    .map(FileStatus::getId)
-                                                                                    .toList())
                                                                    .reportTemplateIds(bulkDownloadRequest.getReportTemplateIds())
                                                                    .downloadFileTypes(bulkDownloadRequest.getDownloadFileTypes())
                                                                    .redactionPreviewColor(bulkDownloadRequest.getRedactionPreviewColor())

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMapper.java

@@ -34,11 +34,13 @@ public interface ComponentMapper {
 
     List<ComponentLogEntryValue> toComponentLogEntries(List<ComponentValue> values);
 
+
     Component toComponent(ComponentLogEntry entry);
 
 
     List<Component> toComponents(List<ComponentLogEntry> entries);
 
+
     ComponentLogEntry toComponentLogEntry(Component component);
 
 

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ComponentLogResource.java

@@ -49,9 +49,7 @@ public interface ComponentLogResource {
     @PostMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Adds overrides for components", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void addOverride(@PathVariable(DOSSIER_ID) String dossierId,
+    void addOverride(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody ComponentsOverrides componentsOverrides);
-                     @PathVariable(FILE_ID) String fileId,
-                     @RequestBody ComponentsOverrides componentsOverrides);
 
 
     @ResponseBody
@@ -59,8 +57,7 @@ public interface ComponentLogResource {
     @GetMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Gets overrides for components", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found")})
-    ComponentsOverrides getOverrides(@PathVariable(DOSSIER_ID) String dossierId,
+    ComponentsOverrides getOverrides(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId);
-                                     @PathVariable(FILE_ID) String fileId);
 
 
     @ResponseBody
@@ -68,8 +65,6 @@ public interface ComponentLogResource {
     @PostMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + "/revert" + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Reverts overrides for components", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void revertOverrides(@PathVariable(DOSSIER_ID) String dossierId,
+    void revertOverrides(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody RevertOverrideRequest revertOverrideRequest);
-                         @PathVariable(FILE_ID) String fileId,
-                         @RequestBody RevertOverrideRequest revertOverrideRequest);
 
 }

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/EntityLogResource.java

@@ -55,7 +55,9 @@ public interface EntityLogResource {
     @GetMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + "/pages", produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Gets the entity log for a fileId with all entities found on the given page numbers", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The dossier / file / entity log is not found.")})
-    EntityLogResponse getEntityLogWithEntriesOnPages(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestParam(value = "pageNumbers", defaultValue = "") List<Integer> pageNumbers);
+    EntityLogResponse getEntityLogWithEntriesOnPages(@PathVariable(DOSSIER_ID) String dossierId,
+                                                     @PathVariable(FILE_ID) String fileId,
+                                                     @RequestParam(value = "pageNumbers", defaultValue = "") List<Integer> pageNumbers);
 
 
     @GetMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + ANALYSIS_NUMBER_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -13,13 +13,15 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.multipart.MultipartFile;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
 
 import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 
@@ -139,11 +141,11 @@ public interface SupportResource {
     DownloadResponse exportFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody FileExchangeExportRequest exportRequest);
 
 
-    @ResponseStatus(value = HttpStatus.NO_CONTENT)
     @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files.")
+    @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. Returns the resulting dossierTemplateId.")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
-    void importFiles(@RequestPart(name = "file") MultipartFile file);
+    ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/BulkDownloadRequest.java

@@ -22,4 +22,5 @@ public class BulkDownloadRequest {
     private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
     private String redactionPreviewColor;
     private List<String> fileIds = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentOverrideList.java

@@ -19,4 +19,5 @@ public class ComponentOverrideList {
     String fileId;
 
     List<Component> componentOverrides = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/DossierAttributeDefinitionList.java

@@ -15,4 +15,5 @@ import lombok.NoArgsConstructor;
 public class DossierAttributeDefinitionList {
 
     private List<DossierAttributeDefinition> dossierAttributeDefinitions = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/DossierStatusDefinitionList.java

@@ -15,4 +15,5 @@ import lombok.NoArgsConstructor;
 public class DossierStatusDefinitionList {
 
     private List<DossierStatusDefinition> dossierStatusDefinitions = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/dossier/DossierACLService.java

@@ -8,6 +8,7 @@ import java.util.stream.Collectors;
 import org.springframework.security.access.prepost.PostFilter;
 import org.springframework.security.acls.domain.ObjectIdentityImpl;
 import org.springframework.security.acls.domain.PrincipalSid;
+import org.springframework.security.acls.model.MutableAcl;
 import org.springframework.security.acls.model.MutableAclService;
 import org.springframework.security.acls.model.NotFoundException;
 import org.springframework.stereotype.Service;
@@ -110,7 +111,7 @@ public class DossierACLService extends AbstractACLService<String> {
     @Transactional
     public void updateDossierACL(Set<String> members, Set<String> approvers, String owner, String dossierId) {
 
-        var acl = getOrCreateACL(dossierId);
+        MutableAcl acl = getOrCreateACL(dossierId);
 
         // naive clear all ace's first
         while (!acl.getEntries().isEmpty()) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/dossier/ViewObjectDossierObjectPermissionService.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Set;
 import java.util.stream.Collectors;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/client/redactionreportservice/RssReportClient.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.persistence.management.v1.processor.client.redactionreportservice;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.redaction.report.v1.api.resource.RSSResource;
+
+@FeignClient(name = "RssReportClient", url = "${redaction-report-service.url}")
+public interface RssReportClient extends RSSResource {
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java

@@ -6,6 +6,7 @@ import static com.knecon.fforesight.service.layoutparser.internal.api.queue.Layo
 
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -193,6 +194,7 @@ public class MessagingConfiguration {
 
 
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrStatusUpdateResponseQueue() {
 
         return QueueBuilder.durable(OCR_STATUS_UPDATE_RESPONSE_QUEUE)
@@ -203,6 +205,7 @@ public class MessagingConfiguration {
 
 
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrStatusUpdateResponseDQL() {
 
         return QueueBuilder.durable(OCR_STATUS_UPDATE_RESPONSE_DQL).build();
@@ -239,6 +242,7 @@ public class MessagingConfiguration {
 
 
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrRequestQueue() {
 
         return QueueBuilder.durable(OCR_REQUEST_QUEUE)
@@ -249,19 +253,20 @@ public class MessagingConfiguration {
                 .build();
     }
 
+
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrResponseQueue() {
 
-        return QueueBuilder.durable(OCR_RESPONSE_QUEUE)
+        return QueueBuilder.durable(OCR_RESPONSE_QUEUE).build();
-                .build();
     }
 
 
     @Bean
+    @ConditionalOnProperty(prefix = "persistence-service", name = "migrateOnly", havingValue = "false")
     public Queue ocrDLQ() {
 
-        return QueueBuilder.durable(OCR_DLQ)
+        return QueueBuilder.durable(OCR_DLQ).build();
-                .build();
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/WebSocketSecurityConfig.java

@@ -73,6 +73,7 @@ public class WebSocketSecurityConfig extends AbstractSecurityWebSocketMessageBro
 
     }
 
+
     private Optional<String> extractTenantId(Message<?> message) {
 
         StompHeaderAccessor sha = StompHeaderAccessor.wrap(message);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierExportService.java

@@ -0,0 +1,70 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.nio.file.Path;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DossierExportService {
+
+    DictionaryPersistenceService dictionaryPersistenceService;
+    FileExportService fileExportService;
+    FileStatusManagementService fileStatusManagementService;
+    EntityTypeExportService entityTypeExportService;
+    ObjectMapper mapper;
+
+
+    @SneakyThrows
+    @Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
+    public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
+
+        List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
+
+        if (!request.dossierIds().contains(dossier.getId()) //
+            && files.stream()
+                    .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
+            // dossier has no files in requested files and dossier not explicitly requested -> don't export it.
+            return;
+        }
+
+        Path dossierFolder = folder.resolve(dossier.getId());
+        archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
+
+        for (FileModel fileEntity : files) {
+            if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
+                continue;
+            }
+            fileExportService.addFileToArchive(archiver, dossierFolder, request, fileEntity);
+        }
+
+        List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
+
+        String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
+
+        for (TypeEntity type : types) {
+            entityTypeExportService.addEntityTypeToArchive(archiver, type, entitiesFolder);
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java

@@ -0,0 +1,84 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DossierImportService {
+
+    DossierCreatorService dossierCreatorService;
+    FileImportService fileImportService;
+    EntityTypeImportService entityTypeImportService;
+
+
+    @Observed(name = "DossierImportService", contextualName = "import-dossier")
+    public void importDossier(FileExchangeImportModel fileExchangeImportModel,
+                              String userId,
+                              Dossier dossierToImport,
+                              TemplateImportInfo templateImportInfo,
+                              List<String> allReportTemplateIds) {
+
+        String dossierId = getDossierForImport(templateImportInfo, dossierToImport, userId, allReportTemplateIds);
+
+        entityTypeImportService.importEntityTypes(templateImportInfo.getDossierTemplateId(),
+                                                  dossierId,
+                                                  fileExchangeImportModel.getDossierTypes()
+                                                          .get(dossierToImport.getId()));
+
+        List<FileExchangeImportModel.FileImport> files = fileExchangeImportModel.getFiles().getOrDefault(dossierToImport.getId(), Collections.emptyList());
+
+        fileImportService.importFilesInParallel(files, userId, templateImportInfo, dossierId);
+    }
+
+
+    private String getDossierForImport(TemplateImportInfo templateImportInfo, Dossier dossier, String userId, List<String> reportTemplateIds) {
+
+        String dossierTemplateId = templateImportInfo.getDossierTemplateId();
+
+        CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
+                .dossierName(dossier.getName())
+                .description(dossier.getDescription())
+                .downloadFileTypes(dossier.getDownloadFileTypes())
+                .dueDate(dossier.getDueDate())
+                .dossierTemplateId(dossierTemplateId)
+                .requestingUser(userId)
+                .reportTemplateIds(reportTemplateIds)
+                .build();
+
+        Dossier importedDossier = null;
+        int retries = 0;
+        while (importedDossier == null && retries < 10) {
+            try {
+                importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
+            } catch (ConflictException e) {
+                retries++;
+                request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
+            }
+        }
+        if (importedDossier == null) {
+            throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
+        }
+        return importedDossier.getId();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java

@@ -123,6 +123,7 @@ public class DossierTemplateExportService {
         });
     }
 
+
     @Observed(name = "DossierTemplateExportService", contextualName = "export-dossier-template")
     public long createDownloadArchive(ExportDownloadMessage downloadJob) throws IOException {
 
@@ -142,6 +143,7 @@ public class DossierTemplateExportService {
 
     }
 
+
     @Observed(name = "DossierTemplateExportService", contextualName = "write-dossier-template-to-archive")
     public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) throws IOException {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateImportService.java

@@ -67,6 +67,7 @@ import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -121,6 +122,7 @@ public class DossierTemplateImportService {
     }
 
 
+    @Observed(name = "DossierTemplateImportService", contextualName = "import-template")
     public TemplateImportInfo importDossierTemplate(ImportTemplateResult request) {
 
         long start = System.currentTimeMillis();

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/EntityTypeExportService.java

@@ -41,6 +41,7 @@ public class EntityTypeExportService {
     ObjectMapper mapper;
     EntryPersistenceService entryPersistenceService;
 
+
     @Observed(name = "EntityTypeExportService", contextualName = "export-entity-type")
     public void addEntityTypeToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, TypeEntity typeEntity, String folder) throws JsonProcessingException {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/EntityTypeImportService.java

@@ -96,6 +96,9 @@ public class EntityTypeImportService {
     @Observed(name = "EntityTypeImportService", contextualName = "import-entity-type")
     public void importEntityTypes(String dossierTemplateId, String dossierId, EntityTypeImportModel importModel) {
 
+        if (importModel == null) {
+            return;
+        }
         enrichObservation(importModel);
 
         for (Type type : importModel.getTypes()) {
@@ -168,7 +171,7 @@ public class EntityTypeImportService {
     private void addEntries(Map<String, List<String>> entries, Map<String, List<String>> deleteEntries, String typeId, String typeName, DictionaryEntryType dictionaryType) {
 
         if (entries != null && !entries.isEmpty() && entries.get(typeName) != null && !entries.get(typeName).isEmpty()) {
-            dictionaryManagementService.addEntries(typeId, entries.get(typeName), true, true, dictionaryType);
+            dictionaryManagementService.addEntries(typeId, entries.get(typeName), true, true, dictionaryType, true);
         } else { // no entries, delete current entries
             List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryType, null)
                     .stream()

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeArchivalService.java

@@ -0,0 +1,38 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.FileExchangeArchiveReader;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
+import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FileExchangeArchivalService {
+
+    public static final long SIZE_THRESHOLD = 10000000000L; // 10GB
+    FileManagementServiceSettings settings;
+
+
+    @SneakyThrows
+    @Observed(name = "FileExchangeImportService", contextualName = "read-import-archive")
+    public FileExchangeImportModel readFileExchangeArchive(byte[] archive, String userId) {
+
+        FileExchangeArchiveReader fileExchangeArchiveReader = new FileExchangeArchiveReader(userId);
+        try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
+            zipEntryIterator.forEachRemaining(fileExchangeArchiveReader::handleZipEntryData);
+        }
+        return fileExchangeArchiveReader.buildResult();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java

@@ -3,28 +3,17 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
@@ -32,15 +21,11 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
-import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -53,16 +38,12 @@ public class FileExchangeExportService {
     DossierTemplateExportService dossierTemplateExportService;
     DossierTemplatePersistenceService dossierTemplatePersistenceService;
     DossierManagementService dossierManagementService;
-    FileStatusManagementService fileStatusManagementService;
     FileManagementStorageService storageService;
     DownloadStatusPersistenceService downloadStatusPersistenceService;
-    ManualChangesExportService manualChangesExportService;
+
-    DictionaryPersistenceService dictionaryPersistenceService;
-    ComponentLogService componentLogService;
-    ObjectMapper mapper;
     RabbitTemplate rabbitTemplate;
-    EntityTypeExportService entityTypeExportService;
     DossierIdFileIdRequestValidator requestValidator;
+    DossierExportService dossierExportService;
 
 
     @Observed(name = "FileExchangeExportService", contextualName = "prepare-export")
@@ -116,7 +97,7 @@ public class FileExchangeExportService {
                     continue;
                 }
 
-                addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
+                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
             }
 
             storeZipFile(downloadJob.getStorageId(), archiver);
@@ -126,125 +107,12 @@ public class FileExchangeExportService {
     }
 
 
-    @SneakyThrows
-    @Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
-    public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
-
-        List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
-
-        if (!request.dossierIds().contains(dossier.getId()) //
-            && files.stream()
-                    .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
-            // dossier has no files in requested files and dossier not explicitly requested -> don't export it.
-            return;
-        }
-
-        Path dossierFolder = folder.resolve(dossier.getId());
-        archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
-
-        for (FileModel fileEntity : files) {
-            if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
-                continue;
-            }
-            addFileToArchive(archiver, dossierFolder, request, fileEntity);
-        }
-
-        List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
-
-        String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
-
-        for (TypeEntity type : types) {
-            entityTypeExportService.addEntityTypeToArchive(archiver, type, entitiesFolder);
-        }
-    }
-
-
-    @SneakyThrows
-    @Observed(name = "FileExchangeExportService", contextualName = "export-file")
-    public void addFileToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, FileModel file) {
-
-        Path fileFolder = folder.resolve(file.getId());
-
-        if (!request.excludeAnalysisLogs()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
-                                                                        mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
-            if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
-                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                            buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
-                                                                            mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
-                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                            buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
-                                                                            mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
-            }
-        }
-
-        archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, FileExchangeNames.FILE_STATUS), mapper.writeValueAsBytes(file)));
-
-        if (!request.excludeLayoutFiles()) {
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.STRUCTURE);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TEXT);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
-        }
-
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.HIGHLIGHTS);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TABLES);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMAGES);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VISUAL_LAYOUT);
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMPORTED);
-
-        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.ORIGIN);
-        if (!request.originFileOnly()) {
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.UNTOUCHED);
-            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VIEWER);
-        }
-
-        if (!request.excludeFileAttributes()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.FILE_ATTRIBUTES),
-                                                                        mapper.writeValueAsBytes(file.getFileAttributes())));
-        }
-
-        if (!request.excludeManualRedactions()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.MANUAL_REDACTIONS),
-                                                                        mapper.writeValueAsBytes(manualChangesExportService.export(file.getId()))));
-        }
-    }
-
-
-    private static String buildFileName(FileModel file, String name) {
-
-        return buildFileName(file.getId(), name);
-    }
-
-
     public static String buildFileName(String fileId, String name) {
 
         return fileId + "." + name;
     }
 
 
-    @SneakyThrows
-    private void addArchiveModelForStorageFile(FileSystemBackedArchiver archiver, FileModel file, Path fileFolder, FileExchangeNames.Definition definition) {
-
-        String storageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), definition.fileType());
-        if (!storageService.objectExists(storageId)) {
-            log.debug("File {} not found in storage with id {}, skipping...", definition.fileType(), storageId);
-            return;
-        }
-        log.debug("Adding file {} from storage with id {} to archive.", definition.fileType(), storageId);
-
-        byte[] data = storageService.getObject(TenantContext.getTenantId(), storageId).readAllBytes();
-        FileSystemBackedArchiver.ArchiveModel archiveModel = new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, definition.getName()), data);
-        archiver.addEntry(archiveModel);
-    }
-
-
     @Observed(name = "FileExchangeExportService", contextualName = "store-archive")
     private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeImportService.java

@@ -1,37 +1,23 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
 
-import java.io.FileInputStream;
-import java.nio.file.Files;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
-import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.FileExchangeArchiveReader;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
-import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
-import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 
-import io.micrometer.common.KeyValue;
 import io.micrometer.observation.ObservationRegistry;
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -41,25 +27,19 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class FileExchangeImportService {
 
-    public static final long SIZE_THRESHOLD = 10000000000L; // 10GB
-
-    FileManagementServiceSettings settings;
     DossierTemplateImportService dossierTemplateImportService;
-    EntityTypeImportService entityTypeImportService;
+    DossierImportService dossierImportService;
-    DossierCreatorService dossierCreatorService;
+    FileExchangeArchivalService fileExchangeArchivalService;
-    FileImportService fileImportService;
-    FileManagementStorageService storageService;
-    ComponentLogService componentLogService;
     ReportTemplatePersistenceService reportTemplateService;
     ObservationRegistry registry;
 
 
     @Observed(name = "FileExchangeImportService", contextualName = "import-files")
-    public String importFileExchangeArchive(String userId, byte[] archive) {
+    public ImportResponse importFileExchangeArchive(String userId, byte[] archive) {
 
         long start = System.currentTimeMillis();
         log.info("Starting file import for user {}", userId);
-        FileExchangeImportModel fileExchangeImportModel = readFileExchangeArchive(archive, userId);
+        FileExchangeImportModel fileExchangeImportModel = fileExchangeArchivalService.readFileExchangeArchive(archive, userId);
         log.info("Parsed file import archive with {} dossiers and {} files",
                  fileExchangeImportModel.getDossiers().size(),
                  fileExchangeImportModel.getFiles().values()
@@ -70,7 +50,29 @@ public class FileExchangeImportService {
 
         enrichObservation(userId, archive, fileExchangeImportModel);
 
-        return dossierTemplateId;
+        return new ImportResponse(dossierTemplateId);
+    }
+
+
+    private String importFileExchangeModel(FileExchangeImportModel fileExchangeImportModel, String userId) {
+
+        ImportTemplateResult importTemplateResult = fileExchangeImportModel.getImportTemplateResult();
+
+        if (importTemplateResult == null) {
+            throw new BadRequestException("DossierTemplate not present in import archive.");
+        }
+
+        TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importTemplateResult);
+
+        List<String> allReportTemplateIds = reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
+                .stream()
+                .map(ReportTemplateEntity::getTemplateId)
+                .toList();
+
+        for (Dossier dossierToImport : fileExchangeImportModel.getDossiers()) {
+            dossierImportService.importDossier(fileExchangeImportModel, userId, dossierToImport, templateImportInfo, allReportTemplateIds);
+        }
+        return templateImportInfo.getDossierTemplateId();
     }
 
 
@@ -88,108 +90,4 @@ public class FileExchangeImportService {
         }
     }
 
-
-    @Observed(name = "FileExchangeImportService", contextualName = "write-import-data-to-storage")
-    private String importFileExchangeModel(FileExchangeImportModel fileExchangeImportModel, String userId) {
-
-        TemplateImportInfo templateImportInfo = getDossierTemplateForImport(fileExchangeImportModel);
-
-        List<String> allReportTemplateIds = reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
-                .stream()
-                .map(ReportTemplateEntity::getTemplateId)
-                .toList();
-
-        for (Dossier dossierToImport : fileExchangeImportModel.getDossiers()) {
-            String dossierId = getDossierForImport(templateImportInfo, dossierToImport, userId, allReportTemplateIds);
-
-            entityTypeImportService.importEntityTypes(templateImportInfo.getDossierTemplateId(),
-                                                      dossierId,
-                                                      fileExchangeImportModel.getDossierTypes()
-                                                              .get(dossierToImport.getId()));
-
-            List<FileExchangeImportModel.FileImport> files = fileExchangeImportModel.getFiles().getOrDefault(dossierToImport.getId(), Collections.emptyList());
-
-            for (FileExchangeImportModel.FileImport file : files) {
-                // separate service for transactions
-                String fileId = fileImportService.saveFileToDb(userId, file, dossierId, templateImportInfo);
-
-                file.getFilesToUpload()
-                        .forEach(fileToUpload -> {
-                            try (FileInputStream in = new FileInputStream(fileToUpload.file().toFile())) {
-                                storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileToUpload.fileType()), in);
-                                Files.deleteIfExists(fileToUpload.file());
-                            } catch (Exception e) {
-                                throw new RuntimeException(e);
-                            }
-                        });
-
-                storageService.saveEntityLog(dossierId, fileId, file.getEntityLog());
-                if (file.getComponentLog() != null) {
-                    storageService.saveComponentLog(dossierId, fileId, file.getComponentLog());
-                }
-                if (file.getOverrides() != null) {
-                    file.getOverrides()
-                            .forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
-                }
-            }
-
-        }
-        return templateImportInfo.getDossierTemplateId();
-    }
-
-
-    private String getDossierForImport(TemplateImportInfo templateImportInfo, Dossier dossier, String userId, List<String> reportTemplateIds) {
-
-        String dossierTemplateId = templateImportInfo.getDossierTemplateId();
-
-        CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
-                .dossierName(dossier.getName())
-                .description(dossier.getDescription())
-                .downloadFileTypes(dossier.getDownloadFileTypes())
-                .dueDate(dossier.getDueDate())
-                .dossierTemplateId(dossierTemplateId)
-                .requestingUser(userId)
-                .reportTemplateIds(reportTemplateIds)
-                .build();
-
-        Dossier importedDossier = null;
-        int retries = 0;
-        while (importedDossier == null && retries < 10) {
-            try {
-                importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
-            } catch (ConflictException e) {
-                retries++;
-                request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
-            }
-        }
-        if (importedDossier == null) {
-            throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
-        }
-        return importedDossier.getId();
-    }
-
-
-    private TemplateImportInfo getDossierTemplateForImport(FileExchangeImportModel fileExchangeImportModel) {
-
-        ImportTemplateResult importTemplateResult = fileExchangeImportModel.getImportTemplateResult();
-
-        if (importTemplateResult == null) {
-            throw new BadRequestException("DossierTemplate not present in import archive.");
-        }
-
-        return dossierTemplateImportService.importDossierTemplate(importTemplateResult);
-    }
-
-
-    @SneakyThrows
-    @Observed(name = "FileExchangeImportService", contextualName = "read-import-archive")
-    private FileExchangeImportModel readFileExchangeArchive(byte[] archive, String userId) {
-
-        FileExchangeArchiveReader fileExchangeArchiveReader = new FileExchangeArchiveReader(userId);
-        try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
-            zipEntryIterator.forEachRemaining(fileExchangeArchiveReader::handleZipEntryData);
-        }
-        return fileExchangeArchiveReader.buildResult();
-    }
-
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExportService.java

@@ -0,0 +1,122 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.nio.file.Path;
+
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
+import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FileExportService {
+
+    ManualChangesExportService manualChangesExportService;
+    ComponentLogService componentLogService;
+    ObjectMapper mapper;
+    FileManagementStorageService storageService;
+
+
+    @SneakyThrows
+    @Observed(name = "FileExchangeExportService", contextualName = "export-file")
+    public void addFileToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, FileModel file) {
+
+        Path fileFolder = folder.resolve(file.getId());
+
+        if (!request.excludeAnalysisLogs()) {
+            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                        buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
+                                                                        mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
+            if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
+                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                            buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
+                                                                            mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
+                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                            buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
+                                                                            mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
+            }
+        }
+
+        archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, FileExchangeNames.FILE_STATUS), mapper.writeValueAsBytes(file)));
+
+        if (!request.excludeLayoutFiles()) {
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.STRUCTURE);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TEXT);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
+        }
+
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.HIGHLIGHTS);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.TABLES);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMAGES);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VISUAL_LAYOUT);
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.IMPORTED);
+
+        addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.ORIGIN);
+        if (!request.originFileOnly()) {
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.UNTOUCHED);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.VIEWER);
+        }
+
+        if (!request.excludeFileAttributes()) {
+            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                        buildFileName(file, FileExchangeNames.FILE_ATTRIBUTES),
+                                                                        mapper.writeValueAsBytes(file.getFileAttributes())));
+        }
+
+        if (!request.excludeManualRedactions()) {
+            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                        buildFileName(file, FileExchangeNames.MANUAL_REDACTIONS),
+                                                                        mapper.writeValueAsBytes(manualChangesExportService.export(file.getId()))));
+        }
+    }
+
+
+    private static String buildFileName(FileModel file, String name) {
+
+        return buildFileName(file.getId(), name);
+    }
+
+
+    public static String buildFileName(String fileId, String name) {
+
+        return fileId + "." + name;
+    }
+
+
+    @SneakyThrows
+    private void addArchiveModelForStorageFile(FileSystemBackedArchiver archiver, FileModel file, Path fileFolder, FileExchangeNames.Definition definition) {
+
+        String storageId = StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), definition.fileType());
+        if (!storageService.objectExists(storageId)) {
+            log.debug("File {} not found in storage with id {}, skipping...", definition.fileType(), storageId);
+            return;
+        }
+        log.debug("Adding file {} from storage with id {} to archive.", definition.fileType(), storageId);
+
+        byte[] data = storageService.getObject(TenantContext.getTenantId(), storageId).readAllBytes();
+        FileSystemBackedArchiver.ArchiveModel archiveModel = new FileSystemBackedArchiver.ArchiveModel(fileFolder, buildFileName(file, definition.getName()), data);
+        archiver.addEntry(archiveModel);
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileImportPersistenceService.java

@@ -0,0 +1,66 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileEntityMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
+
+import io.micrometer.observation.annotation.Observed;
+import jakarta.transaction.Transactional;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class FileImportPersistenceService {
+
+    ManualChangesImportService manualChangesImportService;
+    FileRepository fileRepository;
+
+
+    @Transactional
+    @Observed(name = "FileImportPersistenceService", contextualName = "import-file-to-db")
+    public synchronized String saveFileToDb(String userId,
+                                            FileExchangeImportModel.FileImport file,
+                                            String dossierId,
+                                            TemplateImportInfo templateImportInfo) { // synchronized as this is being called in an async block. Might lock up DB otherwise.
+
+        String fileId = createFile(dossierId, userId, file, templateImportInfo.getIdMapping());
+
+        manualChangesImportService.importManualChanges(file.getManualChanges(), fileId, userId);
+
+        return fileId;
+    }
+
+
+    private String createFile(String dossierId, String userid, FileExchangeImportModel.FileImport file, Map<String, String> fileAttributeConfigMap) {
+
+        FileModel fileModel = file.getFileModel();
+
+        fileModel.setId(UploadService.generateFileId(fileModel.getFilename(), dossierId));
+
+        FileEntity fileEntity = MagicConverter.convert(fileModel, FileEntity.class, new FileEntityMapper(fileAttributeConfigMap));
+
+        fileEntity.setAssignee(userid);
+        fileEntity.setLastReviewer(userid);
+        fileEntity.setLastApprover(userid);
+        fileEntity.setUploader(userid);
+
+        fileEntity.setDossierId(dossierId);
+
+        fileRepository.save(fileEntity);
+
+        return fileEntity.getId();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileImportService.java

@@ -1,57 +1,80 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
 
-import java.util.Map;
+import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.FileExchangeImportModel;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
-import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
+import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
-import com.iqser.red.service.persistence.management.v1.processor.utils.FileEntityMapper;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.knecon.fforesight.tenantcommons.TenantContext;
-import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 
-import jakarta.transaction.Transactional;
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class FileImportService {
 
-
+    FileImportPersistenceService fileImportPersistenceService;
-    ManualChangesImportService manualChangesImportService;
+    FileManagementStorageService storageService;
-    FileRepository fileRepository;
+    ComponentLogService componentLogService;
 
 
-    @Transactional
+    @Observed(name = "FileImportService", contextualName = "import-files-in-parallel")
-    public String saveFileToDb(String userId, FileExchangeImportModel.FileImport file, String dossierId, TemplateImportInfo templateImportInfo) {
+    public void importFilesInParallel(List<FileExchangeImportModel.FileImport> files, String userId, TemplateImportInfo templateImportInfo, String dossierId) {
 
-        String fileId = createFile(dossierId, userId, file, templateImportInfo.getIdMapping());
+        List<CompletableFuture<Void>> fileImportFutures = new LinkedList<>();
 
-        manualChangesImportService.importManualChanges(file.getManualChanges(), fileId);
+        String tenantId = TenantContext.getTenantId();
 
-        return fileId;
+        for (FileExchangeImportModel.FileImport file : files) {
+            fileImportFutures.add(CompletableFuture.supplyAsync(() -> {
+                TenantContext.setTenantId(tenantId);
+                importFile(userId, templateImportInfo, file, dossierId);
+                return null;
+            }));
+        }
+        fileImportFutures.forEach(CompletableFuture::join);
     }
 
 
-    private String createFile(String dossierId, String userid, FileExchangeImportModel.FileImport file, Map<String, String> fileAttributeConfigMap) {
+    @Observed(name = "FileImportService", contextualName = "import-file")
+    public void importFile(String userId, TemplateImportInfo templateImportInfo, FileExchangeImportModel.FileImport file, String dossierId) {
 
-        FileModel fileModel = file.getFileModel();
+        // separate service for transactions
+        String fileId = fileImportPersistenceService.saveFileToDb(userId, file, dossierId, templateImportInfo);
 
-        fileModel.setId(UploadService.generateFileId(fileModel.getFilename(), dossierId));
+        file.getFilesToUpload()
-
+                .forEach(fileToUpload -> {
-        FileEntity fileEntity = MagicConverter.convert(fileModel, FileEntity.class, new FileEntityMapper(fileAttributeConfigMap));
+                    try (FileInputStream in = new FileInputStream(fileToUpload.file().toFile())) {
-
+                        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileToUpload.fileType()), in);
-        fileEntity.setDossierId(dossierId);
+                        Files.deleteIfExists(fileToUpload.file());
-
+                    } catch (Exception e) {
-        fileRepository.save(fileEntity);
+                        throw new RuntimeException(e);
-
+                    }
-        return fileEntity.getId();
+                });
+        if (file.getEntityLog() != null) {
+            storageService.saveEntityLog(dossierId, fileId, file.getEntityLog());
+        }
+        if (file.getComponentLog() != null) {
+            storageService.saveComponentLog(dossierId, fileId, file.getComponentLog());
+        }
+        if (file.getOverrides() != null) {
+            file.getOverrides()
+                    .forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
+        }
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/ManualChangesExportService.java

@@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
 
+import io.micrometer.observation.annotation.Observed;
 import jakarta.transaction.Transactional;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -33,6 +34,7 @@ public class ManualChangesExportService {
 
 
     @Transactional
+    @Observed(name = "ManualChangesExportService", contextualName = "export-manual-changes")
     public ManualChangesExportModel export(String fileId) {
 
         ManualChangesExportModel exportModel = new ManualChangesExportModel();

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/ManualChangesImportService.java

@@ -13,6 +13,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository;
 
+import io.micrometer.observation.annotation.Observed;
 import jakarta.transaction.Transactional;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -36,38 +37,39 @@ public class ManualChangesImportService {
 
 
     @Transactional
-    public void importManualChanges(ManualChangesExportModel exportModel, String fileId) {
+    @Observed(name = "ManualChangesImportService", contextualName = "import-manual-changes")
+    public void importManualChanges(ManualChangesExportModel exportModel, String fileId, String userId) {
 
         FileEntity fileEntity = fileRepository.findById(fileId)
                 .orElseThrow();
         exportModel.getManualAddExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(manualRedactionRepository::saveAndFlush);
 
         exportModel.getManualForceExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(forceRedactionRepository::saveAndFlush);
 
         exportModel.getManualRecategorizationExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(recategorizationRepository::saveAndFlush);
 
         exportModel.getManualRemoveExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(removeRedactionRepository::saveAndFlush);
 
         exportModel.getManualResizeExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(resizeRedactionRepository::saveAndFlush);
 
         exportModel.getManualLegalBasisChangeExportModels()
                 .stream()
-                .map(model -> mapper.toEntity(model, fileEntity))
+                .map(model -> mapper.toEntity(model, fileEntity, userId))
                 .forEach(legalBasisChangeRepository::saveAndFlush);
 
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java

@@ -4,7 +4,6 @@ import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.hibernate.annotations.Fetch;
 import org.springframework.data.annotation.LastModifiedDate;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/AnnotationEntityId.java

@@ -4,7 +4,6 @@ import java.io.Serializable;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Embeddable;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/CommentEntity.java

@@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotat
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.GeneratedValue;
@@ -9,9 +11,6 @@ import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/RectangleEntity.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.annotations;
 
 import jakarta.persistence.Embeddable;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ViewedPageEntity.java

@@ -3,6 +3,8 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotat
 import java.io.Serializable;
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Embeddable;
 import jakarta.persistence.EmbeddedId;
@@ -11,9 +13,6 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.MapsId;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/audit/AuditEntity.java

@@ -4,6 +4,8 @@ import java.time.OffsetDateTime;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
+
 import jakarta.persistence.Basic;
 import jakarta.persistence.Column;
 import jakarta.persistence.Convert;
@@ -13,9 +15,6 @@ import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ApplicationConfigurationEntity.java

@@ -4,7 +4,6 @@ import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ColorsEntity.java

@@ -8,7 +8,6 @@ import jakarta.persistence.Id;
 import jakarta.persistence.PrePersist;
 import jakarta.persistence.PreUpdate;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java

@@ -6,7 +6,6 @@ import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java

@@ -6,7 +6,6 @@ import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DigitalSignatureEntity.java

@@ -1,13 +1,12 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.Lob;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DigitalSignatureKmsEntity.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
+
 import jakarta.persistence.Basic;
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
@@ -7,9 +9,6 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Lob;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DigitalSignatureTypeEntity.java

@@ -1,15 +1,14 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.DigitalSignatureType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
 import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.DigitalSignatureType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/FileAttributesGeneralConfigurationEntity.java

@@ -1,15 +1,14 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.MapsId;
 import jakarta.persistence.OneToOne;
 import jakarta.persistence.Table;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisEntity.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configu
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Embeddable;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisMappingEntity.java

@@ -3,11 +3,15 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configu
 import java.util.ArrayList;
 import java.util.List;
 
-import jakarta.persistence.*;
-
 import org.hibernate.annotations.Fetch;
 import org.hibernate.annotations.FetchMode;
 
+import jakarta.persistence.Column;
+import jakarta.persistence.ElementCollection;
+import jakarta.persistence.Entity;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java

@@ -2,6 +2,9 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configu
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
@@ -9,10 +12,6 @@ import jakarta.persistence.Id;
 import jakarta.persistence.JoinColumn;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/WatermarkEntity.java

@@ -2,6 +2,11 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configu
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.HorizontalTextAlignment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.VerticalTextAlignment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
@@ -13,12 +18,6 @@ import jakarta.persistence.Id;
 import jakarta.persistence.JoinColumn;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.HorizontalTextAlignment;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.VerticalTextAlignment;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/ComponentDefinitionEntity.java

@@ -4,8 +4,6 @@ import java.time.OffsetDateTime;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
 import jakarta.persistence.UniqueConstraint;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeConfigEntity.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
@@ -7,9 +9,6 @@ import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeEntity.java

@@ -7,7 +7,6 @@ import jakarta.persistence.Embeddable;
 import jakarta.persistence.EmbeddedId;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java

@@ -6,6 +6,17 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.hibernate.annotations.Fetch;
+import org.hibernate.annotations.FetchMode;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
+import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
+import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierVisibility;
+
 import jakarta.persistence.CascadeType;
 import jakarta.persistence.Column;
 import jakarta.persistence.Convert;
@@ -19,18 +30,6 @@ import jakarta.persistence.ManyToMany;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.Table;
-
-import org.hibernate.annotations.Fetch;
-import org.hibernate.annotations.FetchMode;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierVisibility;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierStatusEntity.java

@@ -3,6 +3,9 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier
 import java.util.ArrayList;
 import java.util.List;
 
+import org.hibernate.annotations.Fetch;
+import org.hibernate.annotations.FetchMode;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
@@ -11,10 +14,6 @@ import jakarta.persistence.JoinColumn;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.Table;
-
-import org.hibernate.annotations.Fetch;
-import org.hibernate.annotations.FetchMode;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeConfigEntity.java

@@ -1,5 +1,8 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
@@ -7,10 +10,6 @@ import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeEntity.java

@@ -7,7 +7,6 @@ import jakarta.persistence.Embeddable;
 import jakarta.persistence.EmbeddedId;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/ReportTemplateEntity.java

@@ -4,6 +4,8 @@ import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
@@ -11,9 +13,6 @@ import jakarta.persistence.Id;
 import jakarta.persistence.ManyToMany;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadRedactionFileStatusEntity.java

@@ -1,17 +1,21 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.download;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
 import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadRedactionFileDetailsConverter;
 
-import jakarta.persistence.*;
+import jakarta.persistence.Column;
+import jakarta.persistence.Convert;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.ArrayList;
-import java.util.List;
-
 @Data
 @Entity
 @Builder

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/index/IndexInformationEntity.java

@@ -2,13 +2,12 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.index;
 
 import java.time.OffsetDateTime;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.SuppressFBWarnings;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/MigrationEntity.java

@@ -6,7 +6,6 @@ import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/SaasMigrationStatusEntity.java

@@ -2,7 +2,12 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.migrati
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
 
-import jakarta.persistence.*;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.EnumType;
+import jakarta.persistence.Enumerated;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationEntity.java

@@ -4,6 +4,8 @@ import java.time.OffsetDateTime;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
+
 import jakarta.persistence.Basic;
 import jakarta.persistence.Column;
 import jakarta.persistence.Convert;
@@ -12,11 +14,7 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
-import jakarta.persistence.SequenceGenerator;
 import jakarta.persistence.Table;
-
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationPreferencesEntity.java

@@ -3,18 +3,17 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.notific
 import java.util.ArrayList;
 import java.util.List;
 
+import org.hibernate.annotations.Fetch;
+import org.hibernate.annotations.FetchMode;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.notification.EmailNotificationType;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.ElementCollection;
 import jakarta.persistence.Entity;
 import jakarta.persistence.FetchType;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
-
-import org.hibernate.annotations.Fetch;
-import org.hibernate.annotations.FetchMode;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.notification.EmailNotificationType;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/permissions/CustomPermissionEntity.java

@@ -7,7 +7,6 @@ import jakarta.persistence.Entity;
 import jakarta.persistence.Id;
 import jakarta.persistence.IdClass;
 import jakarta.persistence.Table;
-
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/jobs/CreateJobsConfiguration.java

@@ -2,8 +2,6 @@ package com.iqser.red.service.persistence.management.v1.processor.jobs;
 
 import java.text.ParseException;
 
-import com.iqser.red.service.persistence.management.v1.processor.service.job.*;
-
 import org.quartz.CronExpression;
 import org.quartz.CronScheduleBuilder;
 import org.quartz.JobBuilder;
@@ -13,6 +11,15 @@ import org.quartz.TriggerBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import com.iqser.red.service.persistence.management.v1.processor.service.job.AnalysisFlagCalculationSchedulerJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.DeletedFilesCleanupJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadCleanupJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.KeyCloakUserSyncJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.SendNotificationEmailJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.job.SyncUserPermissionsJob;
+
 @Configuration
 public class CreateJobsConfiguration {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/LifecycleAspect.java

@@ -0,0 +1,57 @@
+package com.iqser.red.service.persistence.management.v1.processor.lifecycle;
+
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.springframework.amqp.AmqpRejectAndDontRequeueException;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Aspect
+@Component
+@Slf4j
+@RequiredArgsConstructor
+public class LifecycleAspect {
+
+    private final LifecycleManager lifecycleManager;
+
+    private final LifecycleProperties lifecycleProperties;
+
+
+    @Around("@annotation(org.springframework.amqp.rabbit.annotation.RabbitListener) || "
+            + "@annotation(org.springframework.web.bind.annotation.GetMapping) || "
+            + "@annotation(org.springframework.web.bind.annotation.PostMapping) || "
+            + "@annotation(org.springframework.web.bind.annotation.PutMapping) || "
+            + "@annotation(org.springframework.web.bind.annotation.DeleteMapping) || "
+            + "@annotation(org.springframework.web.bind.annotation.RequestMapping))")
+    public Object checkLifecycle(ProceedingJoinPoint joinPoint) throws Throwable {
+
+        String targetClassName = joinPoint.getTarget().getClass().getPackageName();
+        if (!targetClassName.startsWith(lifecycleProperties.getBasePackage())) {
+            return joinPoint.proceed();
+        }
+
+        synchronized (lifecycleManager) {
+            if (!lifecycleManager.isRunning()) {
+                log.info("Application is shutting down, rejecting new messages.");
+                throw new AmqpRejectAndDontRequeueException("Application is shutting down, rejecting new messages.");
+            }
+            lifecycleManager.incrementAndGet();
+        }
+
+        try {
+            return joinPoint.proceed();
+        } finally {
+            int remainingTasks = lifecycleManager.decrementAndGet();
+            synchronized (lifecycleManager) {
+                if (remainingTasks == 0 && !lifecycleManager.isRunning()) {
+                    lifecycleManager.countDown();
+                    log.info("All tasks are done, ready for shutdown.");
+                }
+            }
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/LifecycleManager.java

@@ -0,0 +1,106 @@
+package com.iqser.red.service.persistence.management.v1.processor.lifecycle;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.springframework.context.ApplicationListener;
+import org.springframework.context.SmartLifecycle;
+import org.springframework.context.event.ContextClosedEvent;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class LifecycleManager implements SmartLifecycle, ApplicationListener<ContextClosedEvent> {
+
+    private volatile boolean running; // by default initialized as false
+    private volatile boolean shutdownInitiated; // by default initialized as false
+    private final Object shutdownMonitor = new Object();
+    private final AtomicInteger activeTasks = new AtomicInteger(0);
+    private final CountDownLatch latch = new CountDownLatch(1);
+
+
+    public void countDown() {
+
+        latch.countDown();
+    }
+
+
+    public int incrementAndGet() {
+
+        return activeTasks.incrementAndGet();
+    }
+
+
+    public int decrementAndGet() {
+
+        return activeTasks.decrementAndGet();
+    }
+
+
+    @Override
+    public void start() {
+
+        synchronized (shutdownMonitor) {
+            running = true;
+        }
+    }
+
+
+    @Override
+    public void stop() {
+
+        synchronized (shutdownMonitor) {
+            running = false;
+            if (activeTasks.get() == 0) {
+                latch.countDown(); // No active tasks, release the latch immediately
+            }
+        }
+    }
+
+
+    @Override
+    public boolean isRunning() {
+
+        return running;
+    }
+
+
+    @Override
+    public boolean isAutoStartup() {
+
+        return true;
+    }
+
+
+    @Override
+    public int getPhase() {
+
+        return Integer.MAX_VALUE; // Start this component last and stop it first
+    }
+
+
+    @Override
+    public void onApplicationEvent(ContextClosedEvent event) {
+
+        synchronized (shutdownMonitor) {
+            if (shutdownInitiated) {
+                return; // Avoid multiple shutdown initiations
+            }
+            shutdownInitiated = true;
+        }
+
+        stop();
+        log.info("Context is closing, waiting for ongoing tasks to complete.");
+        try {
+            latch.await(); // Wait for the latch to count down to zero
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            log.error("Shutdown was interrupted", e);
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/LifecycleProperties.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.persistence.management.v1.processor.lifecycle;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@ConfigurationProperties("lifecycle")
+public class LifecycleProperties {
+
+    private String basePackage;
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ManualChangesExportMapper.java

@@ -8,6 +8,12 @@ import org.mapstruct.MappingTarget;
 import org.mapstruct.Mappings;
 import org.mapstruct.factory.Mappers;
 
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualAddExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualForceExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualLegalBasisChangeExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualRecategorizationExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualRemoveExportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualResizeExportModel;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity;
@@ -16,12 +22,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualAddExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualForceExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualLegalBasisChangeExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualRecategorizationExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualRemoveExportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges.ManualResizeExportModel;
 
 @Mapper
 public interface ManualChangesExportMapper {
@@ -53,79 +53,96 @@ public interface ManualChangesExportMapper {
     ManualResizeExportModel toModel(ManualResizeRedactionEntity entity);
 
 
-    ManualLegalBasisChangeEntity toEntity(ManualLegalBasisChangeExportModel model, @Context FileEntity file);
+    ManualLegalBasisChangeEntity toEntity(ManualLegalBasisChangeExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    ManualForceRedactionEntity toEntity(ManualForceExportModel model, @Context FileEntity file);
+    ManualForceRedactionEntity toEntity(ManualForceExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    ManualRedactionEntryEntity toEntity(ManualAddExportModel model, @Context FileEntity file);
+    ManualRedactionEntryEntity toEntity(ManualAddExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    IdRemovalEntity toEntity(ManualRemoveExportModel model, @Context FileEntity file);
+    IdRemovalEntity toEntity(ManualRemoveExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    ManualRecategorizationEntity toEntity(ManualRecategorizationExportModel model, @Context FileEntity file);
+    ManualRecategorizationEntity toEntity(ManualRecategorizationExportModel model, @Context FileEntity file, @Context String userId);
 
 
-    ManualResizeRedactionEntity toEntity(ManualResizeExportModel model, @Context FileEntity file);
+    ManualResizeRedactionEntity toEntity(ManualResizeExportModel model, @Context FileEntity file, @Context String userId);
 
 
     @AfterMapping
     default void setFileStatusAndId(ManualLegalBasisChangeExportModel model,
                                     @MappingTarget ManualLegalBasisChangeEntity.ManualLegalBasisChangeEntityBuilder entity,
-                                    @Context FileEntity file) {
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
     default void setFileStatusAndId(ManualRecategorizationExportModel model,
                                     @MappingTarget ManualRecategorizationEntity.ManualRecategorizationEntityBuilder entity,
-                                    @Context FileEntity file) {
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualAddExportModel model, @MappingTarget ManualRedactionEntryEntity.ManualRedactionEntryEntityBuilder entity, @Context FileEntity file) {
+    default void setFileStatusAndId(ManualAddExportModel model,
+                                    @MappingTarget ManualRedactionEntryEntity.ManualRedactionEntryEntityBuilder entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualResizeExportModel model, @MappingTarget ManualResizeRedactionEntity.ManualResizeRedactionEntityBuilder entity, @Context FileEntity file) {
+    default void setFileStatusAndId(ManualResizeExportModel model,
+                                    @MappingTarget ManualResizeRedactionEntity.ManualResizeRedactionEntityBuilder entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualForceExportModel model, @MappingTarget ManualForceRedactionEntity.ManualForceRedactionEntityBuilder entity, @Context FileEntity file) {
+    default void setFileStatusAndId(ManualForceExportModel model,
+                                    @MappingTarget ManualForceRedactionEntity.ManualForceRedactionEntityBuilder entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity.IdRemovalEntityBuilder entity, @Context FileEntity file) {
+    default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity.IdRemovalEntityBuilder entity, @Context FileEntity file, @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
         entity.id(annotationEntityId);
         entity.fileStatus(file);
+        entity.user(userId);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/OldQueueCleanup.java

@@ -0,0 +1,40 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration;
+
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE;
+
+import org.springframework.amqp.core.AmqpAdmin;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
+
+import lombok.RequiredArgsConstructor;
+
+// This can not run with migration starter service, it needs to be an InitializingBean otherwise it would be executed after queue listeners are initialized
+
+@Component
+@RequiredArgsConstructor
+public class OldQueueCleanup implements InitializingBean {
+
+    private final AmqpAdmin amqpAdmin;
+    private final FileManagementServiceSettings settings;
+
+    public static final String OCR_QUEUE = "ocrQueue";
+    public static final String OCR_DLQ = "ocrDLQ";
+
+
+    @Override
+    public void afterPropertiesSet() {
+
+        // This should only run in post upgrade hook
+        if (settings.isMigrateOnly()) {
+            amqpAdmin.deleteQueue(OCR_QUEUE);
+            amqpAdmin.deleteQueue(OCR_DLQ);
+            amqpAdmin.deleteQueue(OCR_STATUS_UPDATE_RESPONSE_QUEUE);
+            amqpAdmin.deleteQueue(OCR_STATUS_UPDATE_RESPONSE_DQL);
+            amqpAdmin.deleteQueue("ocr_status_update_response_dql");
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java

@@ -50,8 +50,10 @@ public class StorageToMongoCopyService {
             Optional<EntityLog> entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId);
             if (entityLogFromStorage.isPresent()) {
                 log.info("File found, now saving in mongodb");
-                fileManagementStorageService.saveEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLogFromStorage.get());
+                fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId);
+                fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLogFromStorage.get());
                 log.info("Deleting old file from storage");
+                fileManagementStorageService.storeJSONObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG_BAK, entityLogFromStorage.get());
                 fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG);
             }
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ComponentOverride.java

@@ -18,4 +18,5 @@ public class ComponentOverride {
 
     String name;
     List<ComponentLogEntryValue> componentOverrideValues = new ArrayList<>();
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ManualChangesQueryOptions.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.persistence.management.v1.processor.model;
 
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 
 import lombok.Builder;
 import lombok.Getter;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/AnalyseEvent.java

@@ -21,7 +21,9 @@ public class AnalyseEvent {
     private int numberOfPagesToOCR;
     private int numberOfOCRedPages;
 
+
     public AnalyseEvent(String dossierId, String fileId, AnalyseStatus analyseStatus, Integer analysisNumber, OffsetDateTime timestamp) {
+
         this.dossierId = dossierId;
         this.fileId = fileId;
         this.analyseStatus = analyseStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DossierEvent.java

@@ -10,6 +10,8 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 @AllArgsConstructor
 public class DossierEvent {
+
     private String dossierId;
     private DossierEventType dossierEventType;
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DossierEventType.java

@@ -1,5 +1,11 @@
 package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
 
 public enum DossierEventType {
-    CREATE, UPDATE, SOFT_DELETE, HARD_DELETE, UNDELETE, ARCHIVE, UNARCHIVE
+    CREATE,
+    UPDATE,
+    SOFT_DELETE,
+    HARD_DELETE,
+    UNDELETE,
+    ARCHIVE,
+    UNARCHIVE
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DownloadEvent.java

@@ -16,4 +16,5 @@ public class DownloadEvent {
     private String downloadId;
     private String userId;
     private DownloadStatusValue status;
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/FileEvent.java

@@ -15,5 +15,4 @@ public class FileEvent {
     private String fileId;
     private FileEventType fileEventType;
 
-
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/FileEventType.java

@@ -1,5 +1,9 @@
 package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
 
 public enum FileEventType {
-    CREATE, UPDATE, SOFT_DELETE, HARD_DELETE, UNDELETE
+    CREATE,
+    UPDATE,
+    SOFT_DELETE,
+    HARD_DELETE,
+    UNDELETE
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/NotificationEvent.java

@@ -12,4 +12,5 @@ import lombok.NoArgsConstructor;
 public class NotificationEvent {
 
     private String userId;
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java

@@ -174,8 +174,7 @@ public final class ActionRoles {
     public static final String DEPLOYMENT_INFO = "red-deployment-info";
 
     public static final String USE_SUPPORT_CONTROLLER = "red-use-support-controller";
-
+    public static final String IMPORT_FILES = "red-import-files";
-
 
 
     private ActionRoles() {}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ApplicationRoles.java

@@ -33,6 +33,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_REPORT_TEMPLATES;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_RSS;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_TENANTS;
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.IMPORT_FILES;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.MANAGE_ACL_PERMISSIONS;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.MANAGE_USER_PREFERENCES;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.MANAGE_VIEWED_PAGES;
@@ -116,12 +117,19 @@ public final class ApplicationRoles {
 
     public static final Set<String> UNMAPPED_ACTION_ROLES = Sets.newHashSet(UNARCHIVE_DOSSIER, UPDATE_LICENSE, GET_RSS, USE_SUPPORT_CONTROLLER);
 
-    public static final Set<String> KNECON_ADMIN_ACTION_ROLES = Sets.newHashSet(READ_LICENSE, UPDATE_LICENSE, GET_TENANTS, CREATE_TENANT, READ_USERS, READ_ALL_USERS, WRITE_USERS,
+    public static final Set<String> KNECON_SUPPORT_ACTION_ROLES = Sets.newHashSet(READ_LICENSE,
+                                                                                  UPDATE_LICENSE,
+                                                                                  GET_TENANTS,
+                                                                                  CREATE_TENANT,
+                                                                                  READ_USERS,
+                                                                                  READ_ALL_USERS,
+                                                                                  WRITE_USERS,
                                                                                   READ_SMTP_CONFIGURATION,
                                                                                   WRITE_SMTP_CONFIGURATION,
                                                                                   UNARCHIVE_DOSSIER,
                                                                                   USE_SUPPORT_CONTROLLER);
-    public static final Set<String> KNECON_SUPPORT_ACTION_ROLES = Sets.newHashSet(KNECON_ADMIN_ACTION_ROLES);
+
+    public static final Set<String> KNECON_ADMIN_ACTION_ROLES = Sets.union(KNECON_SUPPORT_ACTION_ROLES, Sets.newHashSet(IMPORT_FILES));
 
     public static final Set<String> RED_USER_ACTION_ROLES = Sets.newHashSet(ADD_COMMENT,
                                                                             READ_LICENSE,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java

@@ -117,7 +117,7 @@ public class AnalysisFlagsCalculationService {
                 hasUpdates = true;
             }
 
-            if (entry.getNumberOfComments() > 0) {
+            if (entry.getNumberOfComments() > 0 && !entry.getState().equals(EntryState.REMOVED)) {
                 hasComments = true;
             }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentDefinitionService.java

@@ -113,7 +113,6 @@ public class ComponentDefinitionService {
     @Transactional
     public List<ComponentDefinition> restoreComponents(String dossierTemplateId, List<String> componentIds) {
 
-
         List<ComponentDefinitionEntity> componentDefinitionEntities = componentDefinitionPersistenceService.restoreComponents(dossierTemplateId, componentIds);
         int rank = componentDefinitionPersistenceService.countByDossierTemplateId(dossierTemplateId);
         for (ComponentDefinitionEntity componentDefinitionEntity : componentDefinitionEntities) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingPersistenceService.java

@@ -7,19 +7,17 @@ import java.time.OffsetDateTime;
 import java.util.List;
 
 import org.springframework.dao.DataAccessException;
-import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
 import com.iqser.red.service.persistence.management.v1.processor.entity.ComponentMappingEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ComponentMappingRepository;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
-import jakarta.validation.ConstraintViolationException;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java

@@ -238,6 +238,13 @@ public class DictionaryManagementService {
     @Transactional
     public void addEntries(String typeId, List<String> entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType) {
 
+        addEntries(typeId, entries, removeCurrent, ignoreInvalidEntries, dictionaryEntryType, false);
+    }
+
+
+    @Transactional
+    public void addEntries(String typeId, List<String> entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType, boolean isImport) {
+
         checkForDossierTypeExistenceAndCreate(typeId);
 
         Set<String> cleanEntries = entries.stream()
@@ -258,7 +265,7 @@ public class DictionaryManagementService {
 
         // To check whether the type exists, type should not be added into database implicitly by addEntry.
         Type typeResult = MagicConverter.convert(dictionaryPersistenceService.getType(typeId), Type.class);
-        if (!typeResult.isHasDictionary()) {
+        if (!typeResult.isHasDictionary() && !isImport) {
             throw new BadRequestException("Entity type does not have a dictionary");
         }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java

@@ -132,7 +132,8 @@ public class DictionaryService {
         if (entriesToDelete != null && !entriesToDelete.isEmpty()) {
             deleteGlobalEntries(type, dossierTemplateId, entriesToDelete, dictionaryEntryType);
         }
-        if (entriesToAdd != null && !entriesToAdd.isEmpty() && !entriesToAdd.stream().allMatch(String::isEmpty)) {
+        if (entriesToAdd != null && !entriesToAdd.isEmpty() && !entriesToAdd.stream()
+                .allMatch(String::isEmpty)) {
             addGlobalEntries(type, dossierTemplateId, entriesToAdd, false, dictionaryEntryType);
         }
     }
@@ -150,7 +151,8 @@ public class DictionaryService {
         if (entriesToDelete != null && !entriesToDelete.isEmpty()) {
             deleteDossierEntries(type, dossierTemplateId, entriesToDelete, dossierId, dictionaryEntryType);
         }
-        if (entriesToAdd != null && !entriesToAdd.isEmpty() && !entriesToAdd.stream().allMatch(String::isEmpty)) {
+        if (entriesToAdd != null && !entriesToAdd.isEmpty() && !entriesToAdd.stream()
+                .allMatch(String::isEmpty)) {
             addDossierEntries(type, dossierTemplateId, entriesToAdd, false, dossierId, dictionaryEntryType);
         }
     }
@@ -487,6 +489,7 @@ public class DictionaryService {
         return MagicConverter.convert(colorsService.getColors(dossierTemplateId), Colors.class);
     }
 
+
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DICTIONARY_TYPE + "')")
     public void changeAddToDictionary(String type, String dossierTemplateId, String dossierId, boolean addToDictionary) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierCreatorService.java

@@ -6,6 +6,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 
+import io.micrometer.observation.annotation.Observed;
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -24,6 +25,7 @@ public class DossierCreatorService {
 
 
     @Transactional
+    @Observed(name = "DossierCreatorService", contextualName = "create-dossier-acl")
     public Dossier addDossier(CreateOrUpdateDossierRequest dossierRequest, Set<String> members, Set<String> approvers, String ownerId) {
 
         var dossier = dossierService.addDossier(dossierRequest);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java

@@ -23,6 +23,7 @@ public class DossierIdFileIdRequestValidator {
     DossierManagementService dossierManagementService;
     FileStatusManagementService fileStatusManagementService;
 
+
     @Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request")
     public void validateRequestOrThrow404(String dossierTemplateId, Set<String> dossierIds, Set<String> fileIds) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java

@@ -64,8 +64,9 @@ public class DossierManagementService {
 
         List<FileModel> fileStatuses = fileStatusService.getDossierStatus(dossierId);
         var relevantFileIds = fileStatuses.stream()
-                .filter(fileStatus -> fileStatus.getDeleted() == null).map(FileModel::getId).toList();
+                .filter(fileStatus -> fileStatus.getDeleted() == null)
-
+                .map(FileModel::getId)
+                .toList();
 
         dossierDeletionService.softDeleteDossier(dossierId, relevantFileIds, now);
         fileDeletionService.reindexDeletedFiles(dossierId, relevantFileIds);
@@ -119,6 +120,7 @@ public class DossierManagementService {
 
     }
 
+
     @Transactional
     public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
 
@@ -208,7 +210,10 @@ public class DossierManagementService {
 
         for (String dossierId : dossierIds) {
 
-            List<String> fileIds = fileStatusService.getDossierStatus(dossierId).stream().map(FileModel::getId).collect(Collectors.toList());
+            List<String> fileIds = fileStatusService.getDossierStatus(dossierId)
+                    .stream()
+                    .map(FileModel::getId)
+                    .collect(Collectors.toList());
             dossierDeletionService.hardDeleteDossier(dossierId, fileIds);
             dossierDeletionService.hardDeleteFileDataAndIndexUpdates(dossierId, fileIds);
         }
@@ -221,8 +226,11 @@ public class DossierManagementService {
         for (String dossierId : dossierIds) {
             var dossier = dossierService.getDossierById(dossierId);
             List<FileModel> fileStatuses = fileStatusService.getDossierStatus(dossierId);
-            var relevantFileIds = fileStatuses.stream().filter(fileStatus -> fileStatus.getDeleted() != null && (fileStatus.getDeleted().equals(dossier.getSoftDeletedTime()) || fileStatus.getDeleted()
+            var relevantFileIds = fileStatuses.stream()
-                        .isAfter(dossier.getSoftDeletedTime()))).map(FileModel::getId).collect(Collectors.toList());
+                    .filter(fileStatus -> fileStatus.getDeleted() != null && (fileStatus.getDeleted().equals(dossier.getSoftDeletedTime()) || fileStatus.getDeleted()
+                            .isAfter(dossier.getSoftDeletedTime())))
+                    .map(FileModel::getId)
+                    .collect(Collectors.toList());
 
             dossierDeletionService.undeleteDossier(dossierId, relevantFileIds, dossier.getSoftDeletedTime());
             dossierDeletionService.reindexUndeletedFiles(dossier.getDossierTemplateId(), dossierId, relevantFileIds);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java

@@ -54,6 +54,8 @@ public class DossierService {
             throw new BadRequestException("Dossier template is not active.");
         }
 
+        validateDossierName(createOrUpdateDossierRequest);
+
         try {
             return dossierPersistenceService.insert(createOrUpdateDossierRequest);
         } catch (Exception e) {
@@ -68,6 +70,14 @@ public class DossierService {
     }
 
 
+    private void validateDossierName(CreateOrUpdateDossierRequest createOrUpdateDossierRequest) {
+
+        if (createOrUpdateDossierRequest.getDossierName().length() > 254) {
+            throw new BadRequestException("Dossier name can not be longer than 254 characters");
+        }
+    }
+
+
     public DossierEntity updateDossier(CreateOrUpdateDossierRequest dossierRequest, String dossierId) {
 
         var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierRequest.getDossierTemplateId());
@@ -143,6 +153,7 @@ public class DossierService {
         return dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
     }
 
+
     public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
 
         return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateManagementService.java

@@ -66,7 +66,6 @@ public class DossierTemplateManagementService {
     }
 
 
-
     public DossierTemplate importDossierTemplate(ImportDossierTemplateRequest request) {
 
         String dossierTemplateId = dossierTemplateImportService.importDossierTemplate(request);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java

@@ -13,6 +13,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -34,6 +35,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@@ -339,6 +341,10 @@ public class EntityLogMergeService {
     public void mergeIdToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) {
 
         entityLogEntry.setState(EntryState.IGNORED);
+        //special case, only for local add, other local changes and then remove
+        if (!entityLogEntry.getEngines().isEmpty() && Set.of(Engine.MANUAL).containsAll(entityLogEntry.getEngines())) {
+            entityLogEntry.setState(EntryState.REMOVED);
+        }
         entityLogEntry.getEngines().add(Engine.MANUAL);
         addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(idRemoval));
@@ -394,6 +400,10 @@ public class EntityLogMergeService {
 
         if (!Strings.isNullOrEmpty(recategorization.getLegalBasis())) {
             boolean isHint = isHint(entityLogEntry.getType(), dossier);
+            if ((entityLogEntry.getEntryType().equals(EntryType.IMAGE) || entityLogEntry.getEntryType().equals(EntryType.IMAGE_HINT)) && !entityLogEntry.getManualChanges()
+                    .isEmpty() && determineHintForImages(entityLogEntry.getManualChanges())) {
+                isHint = false;
+            }
             entityLogEntry.setLegalBasis(recategorization.getLegalBasis());
             entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
         }
@@ -442,6 +452,25 @@ public class EntityLogMergeService {
     }
 
 
+    private boolean determineHintForImages(List<ManualChange> manualChanges) {
+
+        boolean hasForce = false;
+        boolean hasRemoveAfterForce = false;
+
+        for (ManualChange change : manualChanges) {
+            if (change.getManualRedactionType() == ManualRedactionType.FORCE) {
+                hasForce = true;
+            }
+            if (hasForce && change.getManualRedactionType() == ManualRedactionType.REMOVE) {
+                hasRemoveAfterForce = true;
+                break;
+            }
+        }
+
+        return hasForce && !hasRemoveAfterForce;
+    }
+
+
     public boolean isHint(String type, DossierEntity dossier) {
 
         String typeId = toTypeId(type, dossier.getDossierTemplateId());

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java

@@ -36,8 +36,13 @@ public class EntityLogMongoWrapperService {
         List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, ids);
         if (includeUnprocessed) {
             DossierEntity dossier = dossierService.getDossierById(dossierId);
-            ManualRedactions unprocessedManualRedactions = manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.unprocessedOnly());
+            ManualRedactions unprocessedManualRedactions = manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.unprocessedOnly(), ids);
-            entityLogEntries = entityLogMergeService.mergeEntityLogEntries(unprocessedManualRedactions, entityLogEntries.stream().map(EntityLogEntry::getId).toList(), dossier, fileId);
+            entityLogEntries = entityLogMergeService.mergeEntityLogEntries(unprocessedManualRedactions,
+                                                                           entityLogEntries.stream()
+                                                                                   .map(EntityLogEntry::getId)
+                                                                                   .toList(),
+                                                                           dossier,
+                                                                           fileId);
         }
         return entityLogEntries;
     }
@@ -47,4 +52,5 @@ public class EntityLogMongoWrapperService {
 
         return String.format("EntityLogEntry does not exist for annotationId ID \"%s\"!", annotationId);
     }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java

@@ -45,6 +45,7 @@ public class EntityLogService {
         return entityLog;
     }
 
+
     @Observed(name = "EntityLogService", contextualName = "get-entity-log")
     public EntityLog getEntityLog(String dossierId, String fileId, boolean includeUnProcessed) {
 
@@ -52,6 +53,8 @@ public class EntityLogService {
         postProcessEntityLog(dossierId, fileId, entityLog, includeUnProcessed);
         return entityLog;
     }
+
+
     @Observed(name = "EntityLogService", contextualName = "get-entity-log")
     public EntityLog getEntityLog(String dossierId, String fileId, List<String> excludedTypes, boolean includeUnProcessed) {