Merge branch 'RE-10691-fp' into 'master'

RED-10691: 500 when approving file in a dossier where dossier owner lost manager role

Closes RE-10691

See merge request redactmanager/persistence-service!930

.gitignore

@@ -33,4 +33,4 @@ gradlew.bat
 gradle/
 
 **/.gradle
-**/build
+**/build

.gitlab-ci.yml

@@ -20,4 +20,5 @@ deploy:
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_BRANCH =~ /^feature/
     - if: $CI_COMMIT_TAG

README.md

@@ -0,0 +1,57 @@
+![badge](https://img.shields.io/badge/license-knecon-brightgreen)
+## Description
+*TODO*
+
+## Table of Contents
+- [Installation](#installation)
+- [Usage](#usage)
+- [License](#license)
+- [Tests](#tests)
+- [API](#api)
+## Installation
+*TODO*
+
+## Usage
+*TODO*
+
+## License
+![badge](https://img.shields.io/badge/license-knecon-brightgreen)<br />
+This application is covered by the proprietary knecon license. 
+
+## Tests
+*TODO*
+
+## API
+
+### Files
+
+The OpenAPI specifications for RedactManager and Documine are located here:<br> `./persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api`
+
+The Redocly configuration file `redocly.yaml` is located in the root directory of the project. It contains the settings for linting the specifications and generating the API documentation. If you run the redocly commands from there, you do not need to specify the path to the configuration file.
+
+The Redocly files are located in the `redocly` directory:
+* The `templates` directory contains the [Handlebars](https://handlebarsjs.com/) templates for RedactManager and Documine, respectively.
+* The `build` directory is the default output directory for the generated API documentation.
+
+### Redocly Installation
+Execute the following command to install the Redocly CLI.
+```bash
+npm install -g @redocly/cli@latest
+```
+
+### Redocly Linting
+Execute the following commands in the root directory of the project to lint the OpenAPI specifications.
+
+Please refer to the Redocly documentation for detailed information on the linting rules.
+
+```bash
+redocly lint redactmanager
+redocly lint documine
+```
+
+### Redocly Docs Generation
+Execute the following commands in the root directory of the project to generate the API documentation.
+```bash
+redocly build-docs redactmanager --output ./redocly/build/redactmanager.html
+redocly build-docs documine --output ./redocly/build/documine.html
+```

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -7,9 +7,10 @@ plugins {
 }
 
 val redactionServiceVersion by rootProject.extra { "4.290.0" }
-val pdftronRedactionServiceVersion by rootProject.extra { "4.81.0" }
+val pdftronRedactionServiceVersion by rootProject.extra { "4.90.0-RED10115.0" }
 val redactionReportServiceVersion by rootProject.extra { "4.81.0" }
 val searchServiceVersion by rootProject.extra { "2.90.0" }
+val documentVersion by rootProject.extra { "4.433.0" }
 
 repositories {
     mavenLocal()
@@ -45,6 +46,8 @@ tasks.named<Test>("test") {
     reports {
         junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
     }
+    minHeapSize = "512m"
+    maxHeapSize = "2048m"
 }
 
 tasks.test {

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ApplicationConfigurationController.java

@@ -9,11 +9,17 @@ import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.ApplicationConfigurationResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
+import jakarta.persistence.Column;
 import jakarta.validation.Valid;
+import lombok.Builder;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -23,14 +29,24 @@ import lombok.extern.slf4j.Slf4j;
 public class ApplicationConfigurationController implements ApplicationConfigurationResource {
 
     private final ApplicationConfigService applicationConfigService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
     @PreAuthorize("hasAuthority('" + WRITE_APP_CONFIG + "')")
     public ApplicationConfig createOrUpdateAppConfig(@Valid @RequestBody ApplicationConfig appConfig) {
 
-        return MagicConverter.convert(applicationConfigService.saveApplicationConfiguration(MagicConverter.convert(appConfig, ApplicationConfigurationEntity.class)),
+        var result = MagicConverter.convert(applicationConfigService.saveApplicationConfiguration(convert(appConfig)),
                                       ApplicationConfig.class);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId("ApplicationConfig")
+                                              .category(AuditCategory.SETTINGS.name())
+                                              .message("Application config has been changed.")
+                                              .build());
+
+        return result;
     }
 
 
@@ -41,4 +57,18 @@ public class ApplicationConfigurationController implements ApplicationConfigurat
         return MagicConverter.convert(applicationConfigService.getApplicationConfig(), ApplicationConfig.class);
     }
 
+
+    private ApplicationConfigurationEntity convert(ApplicationConfig appConfig){
+        var entity = ApplicationConfigurationEntity.builder()
+                .downloadCleanupDownloadFilesHours(appConfig.getDownloadCleanupDownloadFilesHours())
+                .downloadCleanupNotDownloadFilesHours(appConfig.getDownloadCleanupNotDownloadFilesHours())
+                .softDeleteCleanupTime(appConfig.getSoftDeleteCleanupTime())
+                .build();
+
+        if(appConfig.getHardDeleteCleanupRetryTime() != null){
+            entity.setHardDeleteCleanupRetryTime(appConfig.getHardDeleteCleanupRetryTime());
+        }
+        return entity;
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ComponentLogController.java

@@ -1,131 +0,0 @@
-package com.iqser.red.persistence.service.v1.external.api.impl.controller;
-
-import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_RSS;
-import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_REDACTION_LOG;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import org.springframework.http.MediaType;
-import org.springframework.security.access.prepost.PreAuthorize;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RestController;
-
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
-import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
-import com.iqser.red.service.persistence.service.v1.api.external.resource.ComponentLogResource;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentsOverrides;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@RestController
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ComponentLogController implements ComponentLogResource {
-
-    ComponentLogService componentLogService;
-    AccessControlService accessControlService;
-
-
-    @Override
-    @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')")
-    public ComponentLog getComponentLog(String dossierId, String fileId, boolean includeOverrides) {
-
-        accessControlService.checkDossierExistenceAndViewPermissionsToDossier(dossierId);
-        accessControlService.validateFileResourceExistence(fileId);
-
-        return componentLogService.getComponentLog(dossierId, fileId, includeOverrides);
-    }
-
-
-    @Deprecated(forRemoval = true)
-    @PostMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    @PreAuthorize("hasAuthority('" + GET_RSS + "')")
-    public void addOverride(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody ComponentsOverrides componentsOverrides) {
-
-        accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
-        accessControlService.validateFileResourceExistence(fileId);
-
-        if (componentsOverrides.getComponentOverrides() == null || componentsOverrides.getComponentOverrides().isEmpty()) {
-            throw new BadRequestException("Request body cannot be empty!");
-        }
-
-        var componentLog = componentLogService.getComponentLog(dossierId, fileId);
-
-        componentsOverrides.getComponentOverrides()
-                .forEach((k, v) -> {
-                    ComponentLogEntry override;
-                    var entryOptional = componentLog.getComponentLogEntries()
-                            .stream()
-                            .filter(componentLogEntry -> componentLogEntry.getName().equals(k))
-                            .findFirst();
-                    if (entryOptional.isPresent()) {
-                        override = entryOptional.get();
-                        override.getComponentValues()
-                                .forEach(componentLogEntryValue -> componentLogEntryValue.setValue(v));
-                    } else {
-                        ComponentLogEntryValue overrideValue = ComponentLogEntryValue.builder()
-                                .value(v)
-                                .originalValue(v)
-                                .valueDescription("")
-                                .componentRuleId("")
-                                .componentLogEntityReferences(new ArrayList<>())
-                                .build();
-                        override = ComponentLogEntry.builder().name(k).componentValues(List.of(overrideValue)).build();
-                    }
-                    componentLogService.addOverride(dossierId, fileId, override);
-                });
-
-    }
-
-
-    @Deprecated(forRemoval = true)
-    @GetMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @PreAuthorize("hasAuthority('" + GET_RSS + "')")
-    public ComponentsOverrides getOverrides(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
-
-        accessControlService.checkDossierExistenceAndViewPermissionsToDossier(dossierId);
-        accessControlService.validateFileResourceExistence(fileId);
-
-        var overrides = componentLogService.getOverrides(dossierId, fileId);
-
-        Map<String, String> overridesMap = new HashMap<>();
-        overrides.forEach(componentLogEntry -> {
-            Optional<ComponentLogEntryValue> value = componentLogEntry.getComponentValues()
-                    .stream()
-                    .findAny();
-            value.ifPresent(componentLogEntryValue -> overridesMap.put(componentLogEntry.getName(), componentLogEntryValue.getValue()));
-        });
-        return ComponentsOverrides.builder().componentOverrides(overridesMap).build();
-    }
-
-
-    @Deprecated(forRemoval = true)
-    @PostMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + "/revert" + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    @PreAuthorize("hasAuthority('" + GET_RSS + "')")
-    public void revertOverrides(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody RevertOverrideRequest revertOverrideRequest) {
-
-        accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
-        accessControlService.validateFileResourceExistence(fileId);
-
-        if (revertOverrideRequest.getComponents() == null || revertOverrideRequest.getComponents().isEmpty()) {
-            throw new BadRequestException("Request body cannot be empty!");
-        }
-
-        componentLogService.revertOverrides(dossierId, fileId, revertOverrideRequest);
-    }
-
-}

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/CustomPermissionMappingController.java

@@ -11,8 +11,12 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.acl.custom.service.CustomPermissionService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.CustomPermissionMappingResource;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.permission.CustomPermissionMappingModel;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import lombok.RequiredArgsConstructor;
 
@@ -21,6 +25,7 @@ import lombok.RequiredArgsConstructor;
 public class CustomPermissionMappingController implements CustomPermissionMappingResource {
 
     private final CustomPermissionService customPermissionService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
@@ -36,6 +41,14 @@ public class CustomPermissionMappingController implements CustomPermissionMappin
     public void saveCustomPermissionMappings(@PathVariable(TARGET_OBJECT_NAME) String targetObject, @RequestBody List<CustomPermissionMappingModel> customPermissionMappingModels) {
 
         customPermissionService.saveCustomPermissionMappings(targetObject, customPermissionMappingModels);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(targetObject)
+                                              .category(AuditCategory.SETTINGS.name())
+                                              .message("Custom permissions have been changed.")
+                                              .build());
+
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DictionaryController.java

@@ -357,9 +357,9 @@ public class DictionaryController implements DictionaryResource {
     public void changeFlags(@PathVariable(TYPE_PARAMETER_NAME) String type,
                             @PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
                             @PathVariable(value = DOSSIER_ID_PARAMETER_NAME) String dossierId,
-                            @RequestParam(value = "addToDictionary") boolean addToDictionary) {
+                            @RequestParam(value = "addToDictionaryAction") boolean addToDictionaryAction) {
 
-        dictionaryService.changeAddToDictionary(type, dossierTemplateId, dossierId, addToDictionary);
+        dictionaryService.changeAddToDictionary(type, dossierTemplateId, dossierId, addToDictionaryAction);
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierAttributesController.java

@@ -17,6 +17,7 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.DossierAttributeConfigMapper;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
@@ -52,7 +53,7 @@ public class DossierAttributesController implements DossierAttributesResource {
         var result = MagicConverter.convert(dossierAttributeConfigPersistenceService.setDossierAttributesConfig(dossierTemplateId,
                                                                                                                 MagicConverter.convert(dossierAttributesConfig.getDossierAttributeConfigs(),
                                                                                                                                        DossierAttributeConfigEntity.class)),
-                                            DossierAttributeConfig.class);
+                                            DossierAttributeConfig.class, new DossierAttributeConfigMapper());
         auditPersistenceService.insertRecord(AuditRequest.builder()
                                                      .userId(KeycloakSecurity.getUserId())
                                                      .objectId(dossierTemplateId)
@@ -72,7 +73,7 @@ public class DossierAttributesController implements DossierAttributesResource {
         var result = MagicConverter.convert(dossierAttributeConfigPersistenceService.addOrUpdateDossierAttribute(dossierTemplateId,
                                                                                                                  MagicConverter.convert(dossierAttribute,
                                                                                                                                         DossierAttributeConfigEntity.class)),
-                                            DossierAttributeConfig.class);
+                                            DossierAttributeConfig.class, new DossierAttributeConfigMapper());
         auditPersistenceService.insertRecord(AuditRequest.builder()
                                                      .userId(KeycloakSecurity.getUserId())
                                                      .objectId(dossierTemplateId)
@@ -118,7 +119,7 @@ public class DossierAttributesController implements DossierAttributesResource {
     public DossierAttributesConfig getDossierAttributesConfig(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
 
         var result = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
-        return new DossierAttributesConfig(MagicConverter.convert(result, DossierAttributeConfig.class));
+        return new DossierAttributesConfig(MagicConverter.convert(result, DossierAttributeConfig.class, new DossierAttributeConfigMapper()));
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java

@@ -11,6 +11,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -23,7 +24,11 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.model.websocket.DossierEventType;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
 
+import com.iqser.red.service.persistence.management.v1.processor.service.FilterByPermissionsService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.JsonNode;
 import org.apache.commons.lang3.StringUtils;
+
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -76,6 +81,7 @@ public class DossierController implements DossierResource {
 
     private final DossierManagementService dossierManagementService;
     private final UserService userService;
+    private final FilterByPermissionsService filterByPermissionsService;
     private final FileStatusManagementService fileStatusManagementService;
     private final AuditPersistenceService auditPersistenceService;
     private final NotificationPersistenceService notificationPersistenceService;
@@ -106,6 +112,20 @@ public class DossierController implements DossierResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
+    public DossierChangeResponseV2 changesSinceV2(@RequestBody JSONPrimitive<OffsetDateTime> since) {
+
+        DossierChangeResponseV2 changes = dossierManagementService.changesSinceV2(since);
+
+        // filter only viewables
+        changes.setFileChanges(filterByPermissionsService.onlyViewableHavingDossierId(changes.getFileChanges()));
+        changes.setDossierChanges(filterByPermissionsService.onlyViewableHavingDossierId(changes.getDossierChanges()));
+
+        return changes;
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DOSSIER + "') && (#dossierRequest.dossierId == null || hasPermission(#dossierRequest.dossierId, 'Dossier', 'ACCESS_OBJECT') )")
     public ResponseEntity<Dossier> createDossierOrUpdateDossier(@RequestBody DossierRequest dossierRequest) {
@@ -405,6 +425,28 @@ public class DossierController implements DossierResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
+    public JSONPrimitive<Map<String, Dossier>> getDossiersByIds(@RequestBody JSONPrimitive<Set<String>> dossierIds) {
+
+        // filter dossiers based on view
+        var viewableDossierIds = filterByPermissionsService.onlyViewableDossierIds(dossierIds.getValue());
+        // load dossiers
+        var dossiers = dossierManagementService.getDossiersByIds(viewableDossierIds);
+
+        // add attributes and ACL - already filtered before loading
+        enhanceDossiersWithAttributeAndACLData(dossiers,false);
+
+        // build response
+        var responseMap = new LinkedHashMap<String, Dossier>();
+        for (var dossier : dossiers) {
+            responseMap.put(dossier.getId(), dossier);
+        }
+
+        return new JSONPrimitive<>(responseMap);
+    }
+
+
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
     public Dossier getDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
                               @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
@@ -418,70 +460,45 @@ public class DossierController implements DossierResource {
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                                      @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted) {
 
-        var dossiers = dossierManagementService.getAllDossiers(includeArchived, includeDeleted)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getAllDossiers(includeArchived, includeDeleted);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getDossiersForDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
                                                        @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                                                        @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted) {
 
-        var dossiers = dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getSoftDeletedDossiers() {
 
-        var dossiers =  dossierManagementService.getSoftDeletedDossiers()
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getSoftDeletedDossiers();
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiers() {
 
-        var dossiers =  dossierManagementService.getArchivedDossiers()
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getArchivedDossiers();
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiersForDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) {
 
-        var dossiers = dossierManagementService.getArchivedDossiersForDossierTemplateId(dossierTemplateId)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getArchivedDossiersForDossierTemplateId(dossierTemplateId);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
@@ -586,5 +603,31 @@ public class DossierController implements DossierResource {
         return new DossierAttributes(attributeIdToValue);
     }
 
+    private List<Dossier> enhanceDossiersWithAttributeAndACLData(List<Dossier> dossiers) {
+
+            return enhanceDossiersWithAttributeAndACLData(dossiers, true);
+    }
+
+    private List<Dossier> enhanceDossiersWithAttributeAndACLData(List<Dossier> dossiers, boolean filter) {
+
+        // filter first, only load attributes and ACL for viewable dossiers
+        List<Dossier> filteredDossiers = filter ? filterByPermissionsService.onlyViewableDossiers(dossiers) : dossiers;
+
+        // load all attributes at once
+        var attributes = dossierAttributePersistenceService.getDossierAttributes(filteredDossiers.stream().map(Dossier::getId).collect(Collectors.toSet()));
+        var attributesMap = new HashMap<String, List<DossierAttributeEntity>>();
+        for (DossierAttributeEntity attribute : attributes) {
+            attributesMap.computeIfAbsent(attribute.getId().getDossierId(), k -> new ArrayList<>()).add(attribute);
+        }
+
+        for (var dossier : filteredDossiers) {
+            // set attributes
+            dossier.setDossierAttributes(convertDossierAttributes(attributesMap.getOrDefault(dossier.getId(), new ArrayList<>())));
+            // set ACL data
+            dossierACLService.enhanceDossierWithACLData(dossier);
+        }
+        return filteredDossiers;
+    }
+
 }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierStatusController.java

@@ -14,13 +14,17 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.ColorUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.DossierStatusResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -31,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class DossierStatusController implements DossierStatusResource {
 
     private final DossierStatusPersistenceService dossierStatusPersistenceService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
@@ -57,6 +62,15 @@ public class DossierStatusController implements DossierStatusResource {
                                                                                            .rank(dossierStatusRequest.getRank())
                                                                                            .color(dossierStatusRequest.getColor())
                                                                                            .build());
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(dossierStatusRequest.getDossierTemplateId())
+                                              .category(AuditCategory.DOSSIER_TEMPLATE.name())
+                                              .message("Dossier states have been updated.")
+                                              .build());
+
+
         return MagicConverter.convert(response, DossierStatusInfo.class);
     }
 
@@ -91,7 +105,17 @@ public class DossierStatusController implements DossierStatusResource {
     public void deleteDossierStatus(@PathVariable("dossierStatusId") String dossierStatusId,
                                     @RequestParam(value = DOSSIER_STATUS_REPLACE_ID, required = false) String replaceDossierStatusId) {
 
+        var dossierTemplateId = dossierStatusPersistenceService.getDossierStatus(dossierStatusId).getDossierTemplateId();
+
         dossierStatusPersistenceService.deleteDossierStatus(dossierStatusId, replaceDossierStatusId);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(dossierTemplateId)
+                                              .category(AuditCategory.DOSSIER_TEMPLATE.name())
+                                              .message("Dossier state has been deleted.")
+                                              .build());
+
     }
 
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java

@@ -7,23 +7,17 @@ import java.io.BufferedInputStream;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.InputStreamResource;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.context.request.RequestAttributes;
 import org.springframework.web.context.request.RequestContextHolder;
 import org.springframework.web.context.request.ServletRequestAttributes;
 
@@ -34,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.AccessC
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DownloadService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
@@ -76,6 +71,7 @@ public class DownloadController implements DownloadResource {
     private final OneTimeTokenService oneTimeTokenDownloadService;
     private final AccessControlService accessControlService;
     private final FileManagementStorageService fileManagementStorageService;
+    private final FileStatusManagementService fileStatusManagementService;
 
     private final String REPORT_INFO = "/REPORT_INFO.json";
 
@@ -142,7 +138,7 @@ public class DownloadController implements DownloadResource {
         if (StringUtils.isBlank(dossierId)) {
             throw new BadRequestException("Empty dossier id");
         }
-        dossierService.getDossierById(dossierId, true, true);
+        dossierService.getDossierById(dossierId, true, false);
         accessControlService.verifyUserIsDossierOwnerOrApprover(dossierId);
     }
 
@@ -152,16 +148,19 @@ public class DownloadController implements DownloadResource {
         List<FileModel> validFiles = fileStatusService.getDossierStatus(request.getDossierId());
         var fileIds = request.getFileIds();
         if (fileIds != null && !fileIds.isEmpty()) { // validate the ids provided
+            if(fileIds.size() == 1) {
+                fileStatusManagementService.getFileStatus(fileIds.get(0), false);
+            }
             validFiles = validFiles.stream()
                     .filter(f -> fileIds.contains(f.getId()))
                     .collect(Collectors.toList());
             if (validFiles.isEmpty()) {
-                throw new NotFoundException("No file id provided is found");
+                throw new NotFoundException("No provided file id was found");
             }
         } // otherwise consider the files from dossier
 
         var validFilesAndNotProcessed = validFiles.stream()
-                .filter(f -> !(f.getAnalysisVersion() > 0 && f.getNumberOfAnalyses() > 0))
+                .filter(f -> !(f.getAnalysisVersion() > 0 && f.getNumberOfAnalyses() > 0 && !f.isSoftOrHardDeleted()))
                 .collect(Collectors.toList());
         if (!validFilesAndNotProcessed.isEmpty()) {
             throw new BadRequestException("At least a file is in its initial analysis process");

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileAttributesController.java

@@ -15,6 +15,7 @@ import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
 import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileAttributeConfigMapper;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
@@ -61,9 +62,9 @@ public class FileAttributesController implements FileAttributesResource {
         }
         fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
                                                                              MagicConverter.convert(fileAttributesConfig, FileAttributesGeneralConfigurationEntity.class));
-        var result = fileAttributeConfigPersistenceService.setFileAttributesConfig(dossierTemplateId,
-                                                                                   MagicConverter.convert(fileAttributesConfig.getFileAttributeConfigs(),
-                                                                                                          FileAttributeConfigEntity.class));
+        fileAttributeConfigPersistenceService.setFileAttributesConfig(dossierTemplateId,
+                                                                      MagicConverter.convert(fileAttributesConfig.getFileAttributeConfigs(), FileAttributeConfigEntity.class));
+
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierTemplateId)
@@ -74,7 +75,9 @@ public class FileAttributesController implements FileAttributesResource {
                 .filenameMappingColumnHeaderName(fileAttributesConfig.getFilenameMappingColumnHeaderName())
                 .delimiter(fileAttributesConfig.getDelimiter())
                 .encoding(fileAttributesConfig.getEncoding())
-                .fileAttributeConfigs(MagicConverter.convert(result, FileAttributeConfig.class))
+                .fileAttributeConfigs(MagicConverter.convert(fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId),
+                                                             FileAttributeConfig.class,
+                                                             new FileAttributeConfigMapper()))
                 .build();
 
     }
@@ -96,7 +99,7 @@ public class FileAttributesController implements FileAttributesResource {
                                                               dossierTemplateId))
                                               .build());
 
-        return MagicConverter.convert(result, FileAttributeConfig.class);
+        return MagicConverter.convert(result, FileAttributeConfig.class, new FileAttributeConfigMapper());
     }
 
 
@@ -145,7 +148,7 @@ public class FileAttributesController implements FileAttributesResource {
                 .filenameMappingColumnHeaderName(generalConfig.getFilenameMappingColumnHeaderName())
                 .delimiter(generalConfig.getDelimiter())
                 .encoding(generalConfig.getEncoding())
-                .fileAttributeConfigs(MagicConverter.convert(fileAttributeConfigs, FileAttributeConfig.class))
+                .fileAttributeConfigs(MagicConverter.convert(fileAttributeConfigs, FileAttributeConfig.class, new FileAttributeConfigMapper()))
                 .build();
     }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java

@@ -232,7 +232,7 @@ public class FileManagementController implements FileManagementResource {
     public void restoreFiles(@PathVariable(DOSSIER_ID) String dossierId, @RequestBody Set<String> fileIds) {
 
         accessControlService.checkAccessPermissionsToDossier(dossierId);
-        verifyUserIsDossierOwnerOrApproverOrAssignedReviewer(dossierId, fileIds);
+        accessControlService.verifyUserIsDossierOwnerOrApproverOrAssignedReviewer(dossierId, fileIds);
         fileService.undeleteFiles(dossierId, fileIds);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
@@ -266,11 +266,14 @@ public class FileManagementController implements FileManagementResource {
             FileModel fileModel = fileStatusManagementService.getFileStatus(fileId);
 
             if (!fileModel.isExcludedFromAutomaticAnalysis()) {
-                if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) {
-                    reanalysisService.ocrFile(dossierId, fileId, true);
-                } else {
-                    fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
-                }
+
+                // Re- ocr is files with pdftron ocr service is breaking files and not needed with azure.
+                // TODO find a better solution for this.
+//                if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) {
+//                    reanalysisService.ocrFile(dossierId, fileId, true);
+//                } else {
+                    fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
+//                }
             }
 
             auditPersistenceService.audit(AuditRequest.builder()
@@ -286,20 +289,4 @@ public class FileManagementController implements FileManagementResource {
         }
     }
 
-
-    private void verifyUserIsDossierOwnerOrApproverOrAssignedReviewer(String dossierId, Set<String> fileIds) {
-
-        try {
-            accessControlService.verifyUserIsDossierOwnerOrApprover(dossierId);
-        } catch (AccessDeniedException e1) {
-            try {
-                for (String fileId : fileIds) {
-                    accessControlService.verifyUserIsReviewer(dossierId, fileId);
-                }
-            } catch (NotAllowedException e2) {
-                throw new NotAllowedException("User must be dossier owner, approver or assigned reviewer of the file.");
-            }
-        }
-    }
-
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java

@@ -2,6 +2,7 @@ package com.iqser.red.persistence.service.v1.external.api.impl.controller;
 
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_SIMILAR_IMAGES;
 
+import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -11,6 +12,7 @@ import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.commons.spring.ErrorMessage;
 import com.iqser.red.service.persistence.management.v1.processor.service.ImageSimilarityService;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.ImageSimilaritySearchResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchRequest;
@@ -31,10 +33,15 @@ public class ImageSimilaritySearchController implements ImageSimilaritySearchRes
 
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + GET_SIMILAR_IMAGES + "')")
-    public ResponseEntity<ImageSimilaritySearchResponse> getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest) {
+    public ResponseEntity getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest) {
 
         log.info("received similiar image search request {}", imageSimilaritySearchRequest);
-        List<ImageDocument> similarImages = this.imageSimilarityService.findSimilarImages(imageSimilaritySearchRequest.getCentroId(),
+        if (imageSimilaritySearchRequest.getAnnotationId().isEmpty() || (imageSimilaritySearchRequest.getScope().getFileId().isEmpty() && imageSimilaritySearchRequest.getScope()
+                .getTemplateId()
+                .isEmpty() && imageSimilaritySearchRequest.getScope().getDossierId().isEmpty())) {
+            return new ResponseEntity<>(new ErrorMessage(OffsetDateTime.now(), "Required parameter missing"), HttpStatus.BAD_REQUEST);
+        }
+        List<ImageDocument> similarImages = this.imageSimilarityService.findSimilarImages(imageSimilaritySearchRequest.getAnnotationId(),
                                                                                           imageSimilaritySearchRequest.getDistance(),
                                                                                           imageSimilaritySearchRequest.getScope());
         List<String> similarImagesIds = new ArrayList<>();

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/LegalBasisMappingController.java

@@ -5,22 +5,21 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
 
 import java.util.List;
 
-import jakarta.transaction.Transactional;
-
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
-import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.LegalBasisMappingResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
+import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 
 @RestController
@@ -33,9 +32,9 @@ public class LegalBasisMappingController implements LegalBasisMappingResource {
 
     @Override
     @PreAuthorize("hasAuthority('" + WRITE_LEGAL_BASIS + "')")
-    public void deleteLegalBasis(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @RequestBody List<String> legalBasisNames) {
+    public void deleteLegalBasis(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @RequestBody List<String> legalBasisTechnicalNames) {
 
-        legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId, legalBasisNames);
+        legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId, legalBasisTechnicalNames);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierTemplateId)

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java

@@ -6,9 +6,13 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.DO_MANUAL_REDACTION;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_MANUAL_REDACTIONS;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -16,31 +20,48 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.mapper.EntityLogResponseMapper;
 import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
 import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionUndoService;
+import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.PendingEntryFactory;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.ManualRedactionResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.CommentResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntryResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationComments;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.CommentRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionBulkLocalRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationBulkLocalRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionBulkLocalRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ResizeRedactionRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import lombok.AccessLevel;
@@ -57,6 +78,7 @@ public class ManualRedactionController implements ManualRedactionResource {
     static final String FILE_ID = "fileId";
     static final String DOSSIER_ID = "dossierId";
     static final String ANNOTATION_ID = "annotationId";
+
     ManualRedactionService manualRedactionService;
     ManualRedactionUndoService manualRedactionUndoService;
     DossierManagementService dossierManagementService;
@@ -64,9 +86,15 @@ public class ManualRedactionController implements ManualRedactionResource {
     AccessControlService accessControlService;
     CommentService commentService;
     FileStatusManagementService fileStatusManagementService;
+    FileStatusService fileStatusService;
+    EntityLogMongoService entityLogMongoService;
+    PendingEntryFactory pendingEntryFactory;
+    DictionaryPersistenceService dictionaryPersistenceService;
+
+    EntityLogController entityLogController;
+    EntityLogResponseMapper mapper = EntityLogResponseMapper.INSTANCE;
 
 
-    @Override
     @PreAuthorize("hasAuthority('" + DELETE_MANUAL_REDACTION + "')")
     public void undo(@PathVariable(DOSSIER_ID) String dossierId,
                      @PathVariable(FILE_ID) String fileId,
@@ -82,7 +110,6 @@ public class ManualRedactionController implements ManualRedactionResource {
     }
 
 
-    @Override
     @PreAuthorize("hasAuthority('" + DELETE_COMMENT + "')")
     public void undoComment(@PathVariable(DOSSIER_ID) String dossierId,
                             @PathVariable(FILE_ID) String fileId,
@@ -105,7 +132,6 @@ public class ManualRedactionController implements ManualRedactionResource {
     }
 
 
-    @Override
     @PreAuthorize("hasAuthority('" + READ_MANUAL_REDACTIONS + "')")
     public ManualRedactions getManualRedactions(@PathVariable(DOSSIER_ID) String dossierId,
                                                 @PathVariable(FILE_ID) String fileId,
@@ -114,12 +140,12 @@ public class ManualRedactionController implements ManualRedactionResource {
 
         accessControlService.checkDossierExistenceAndViewPermissionsToDossier(dossierId);
         accessControlService.validateFileResourceExistence(fileId);
+
         return manualRedactionService.getManualRedactions(fileId,
                                                           ManualChangesQueryOptions.builder().includeOnlyUnprocessed(unprocessed).includeDictChanges(includeDictChanges).build());
     }
 
 
-    @Override
     @PreAuthorize("hasAuthority('" + READ_MANUAL_REDACTIONS + "')")
     public AnnotationComments getComments(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @PathVariable(ANNOTATION_ID) String annotationId) {
 
@@ -132,7 +158,6 @@ public class ManualRedactionController implements ManualRedactionResource {
     }
 
 
-    @Override
     @PreAuthorize("hasAuthority('" + ADD_COMMENT + "')")
     public CommentResponse addComment(@PathVariable(DOSSIER_ID) String dossierId,
                                       @PathVariable(FILE_ID) String fileId,
@@ -157,23 +182,18 @@ public class ManualRedactionController implements ManualRedactionResource {
     }
 
 
-    @Override
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse addRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                     @PathVariable(FILE_ID) String fileId,
                                                     @RequestBody Set<AddRedactionRequestModel> addRedactionRequests) {
 
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
-        accessControlService.checkAccessPermissionsToDossier(dossierId);
-        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
-        if (addRedactionRequests.stream()
-                .anyMatch(AddRedactionRequestModel::isAddToAllDossiers)) {
-            accessControlService.verifyUserIsApprover(dossierId);
-        } else {
-            accessControlService.verifyUserIsMemberOrApprover(dossierId);
-        }
+        verifyAccessForDossier(dossierId,
+                               fileId,
+                               addRedactionRequests.stream()
+                                       .anyMatch(AddRedactionRequestModel::isAddToAllDossiers));
 
-        List<ManualAddResponse> responseList = manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests, dossier);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests, dossier);
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
                                                                                .objectId(fileId)
@@ -182,31 +202,68 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                                                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
                                                                                .build()));
 
-        return ManualRedactionResponse.builder().manualAddResponses(responseList).build();
+        return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build();
+    }
+
+
+    @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
+    public ManualRedactionResponse addRedactionBulkLocal(String dossierId, String fileId, AddRedactionBulkLocalRequestModel addRedactionRequest) {
+
+        verifyAccess(dossierId, fileId);
+
+        // check if type exists before sending the request to redaction service
+        var dossier = dossierManagementService.getDossierById(dossierId, false, false);
+        dictionaryPersistenceService.getType(TypeIdUtils.toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId()));
+
+        if (!addRedactionRequest.isRectangle()) {
+            fileStatusService.setStatusBulkLocalRedactionsProcessing(fileId, addRedactionRequest);
+
+            EntityLogEntry entityLogEntry = pendingEntryFactory.buildAddRedactionBulkLocalEntry(addRedactionRequest);
+
+            return ManualRedactionResponse.builder()
+                    .manualAnnotationResponses(List.of(ManualAnnotationResponse.builder()
+                                                               .annotationId(manualRedactionService.getNewAnnotationId())
+                                                               .entityLogEntry(entityLogEntry)
+                                                               .build()))
+                    .build();
+        } else {
+            Set<AddRedactionRequestModel> addRedactionRequestModels = addRedactionRequest.getPageNumbers()
+                    .stream()
+                    .map(page -> AddRedactionRequestModel.builder()
+                            .type(addRedactionRequest.getType())
+                            .value(addRedactionRequest.getValue())
+                            .reason(addRedactionRequest.getReason())
+                            .legalBasis(addRedactionRequest.getLegalBasis())
+                            .positions(addRedactionRequest.getPositions()
+                                               .stream()
+                                               .map(rectangle -> Rectangle.createRectangle(rectangle.getTopLeftX(),
+                                                                                           rectangle.getTopLeftY(),
+                                                                                           rectangle.getWidth(),
+                                                                                           rectangle.getHeight(),
+                                                                                           page))
+                                               .toList())
+                            .comment(addRedactionRequest.getComment() != null ? AddCommentRequestModel.builder().text(addRedactionRequest.getComment()).build() : null)
+                            .section(addRedactionRequest.getSection())
+                            .rectangle(true)
+                            .build())
+                    .collect(Collectors.toSet());
+            return addRedactionBulk(dossierId, fileId, addRedactionRequestModels);
+        }
     }
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse removeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                        @PathVariable(FILE_ID) String fileId,
-                                                       @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests,
-                                                       @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                       @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests) {
 
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
-        accessControlService.checkAccessPermissionsToDossier(dossierId);
-        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
-        if (removeRedactionRequests.stream()
-                .anyMatch(RemoveRedactionRequestModel::isRemoveFromAllDossiers)) {
-            accessControlService.verifyUserIsApprover(dossierId);
-        } else {
-            accessControlService.verifyUserIsMemberOrApprover(dossierId);
-        }
+        verifyAccessForDossier(dossierId,
+                               fileId,
+                               removeRedactionRequests.stream()
+                                       .anyMatch(RemoveRedactionRequestModel::isRemoveFromAllDossiers));
 
-        List<ManualAddResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId,
-                                                                                         fileId,
-                                                                                         removeRedactionRequests,
-                                                                                         dossier.getDossierTemplateId(),
-                                                                                         includeUnprocessed);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId, fileId, removeRedactionRequests, dossier.getDossierTemplateId(), true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -216,7 +273,49 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                                                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
                                                                                .build()));
 
-        return ManualRedactionResponse.builder().manualAddResponses(responseList).build();
+        return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build();
+    }
+
+
+    @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
+    public ManualRedactionResponse removeRedactionBulkLocal(String dossierId, String fileId, RemoveRedactionBulkLocalRequestModel removeRedactionRequest) {
+
+        verifyAccess(dossierId, fileId);
+        verifyRequest(removeRedactionRequest.isRectangle(), removeRedactionRequest.getPosition(), removeRedactionRequest.getValue());
+
+        Set<RemoveRedactionRequestModel> removeRedactionRequestModels;
+        FileModel status = fileStatusService.getStatus(fileId);
+
+        Set<EntityLogEntry> entries;
+        if (!status.isExcludedFromAutomaticAnalysis()) {
+            entries = getFilteredEntityLogEntries(dossierId,
+                                                  fileId,
+                                                  removeRedactionRequest.isRectangle(),
+                                                  removeRedactionRequest.getValue(),
+                                                  removeRedactionRequest.isCaseSensitive(),
+                                                  removeRedactionRequest.getOriginTypes(),
+                                                  removeRedactionRequest.getOriginLegalBases(),
+                                                  removeRedactionRequest.getPageNumbers(),
+                                                  removeRedactionRequest.getPosition());
+
+        } else {
+
+            entries = new HashSet<>(mapper.fromLogEntryResponses(getFilteredEntityLogResponses(dossierId,
+                                                                                               fileId,
+                                                                                               removeRedactionRequest.isRectangle(),
+                                                                                               removeRedactionRequest.getValue(),
+                                                                                               removeRedactionRequest.isCaseSensitive(),
+                                                                                               removeRedactionRequest.getOriginTypes(),
+                                                                                               removeRedactionRequest.getOriginLegalBases(),
+                                                                                               removeRedactionRequest.getPageNumbers(),
+                                                                                               removeRedactionRequest.getPosition())));
+        }
+
+        removeRedactionRequestModels = entries.stream()
+                .map(entityLogEntry -> RemoveRedactionRequestModel.builder().annotationId(entityLogEntry.getId()).comment(removeRedactionRequest.getComment()).build())
+                .collect(Collectors.toSet());
+
+        return removeRedactionBulk(dossierId, fileId, removeRedactionRequestModels);
     }
 
 
@@ -225,21 +324,19 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                       @PathVariable(FILE_ID) String fileId,
                                                       @RequestBody Set<ForceRedactionRequestModel> forceRedactionRequests) {
 
-        accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
-        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
-        accessControlService.verifyUserIsMemberOrApprover(dossierId);
+        verifyAccessAndDossierExistence(dossierId, fileId);
 
-        List<ManualAddResponse> responseList = manualRedactionService.addForceRedaction(dossierId, fileId, forceRedactionRequests);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addForceRedaction(dossierId, fileId, forceRedactionRequests);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
                                                                                .objectId(fileId)
                                                                                .category(AuditCategory.DOCUMENT.name())
-                                                                               .message("Skipped annotation was forced to be redacted")
+                                                                               .message("Skipped annotation was forced to be applied")
                                                                                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
                                                                                .build()));
 
-        return ManualRedactionResponse.builder().manualAddResponses(responseList).build();
+        return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build();
     }
 
 
@@ -248,11 +345,9 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                         @PathVariable(FILE_ID) String fileId,
                                                         @RequestBody Set<LegalBasisChangeRequestModel> legalBasisChangeRequests) {
 
-        accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
-        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
-        accessControlService.verifyUserIsMemberOrApprover(dossierId);
+        verifyAccessAndDossierExistence(dossierId, fileId);
 
-        List<ManualAddResponse> responseList = manualRedactionService.addLegalBasisChange(dossierId, fileId, legalBasisChangeRequests);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addLegalBasisChange(dossierId, fileId, legalBasisChangeRequests);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -262,26 +357,19 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                                                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
                                                                                .build()));
 
-        return ManualRedactionResponse.builder().manualAddResponses(responseList).build();
+        return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build();
     }
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse recategorizeBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                     @PathVariable(FILE_ID) String fileId,
-                                                    @RequestBody Set<RecategorizationRequestModel> recategorizationRequests,
-                                                    @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                    @RequestBody Set<RecategorizationRequestModel> recategorizationRequests) {
 
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
-        accessControlService.checkAccessPermissionsToDossier(dossierId);
-        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
-        accessControlService.verifyUserIsMemberOrApprover(dossierId);
+        verifyAccess(dossierId, fileId);
 
-        List<ManualAddResponse> responseList = manualRedactionService.addRecategorization(dossierId,
-                                                                                          fileId,
-                                                                                          dossier,
-                                                                                          recategorizationRequests,
-                                                                                          includeUnprocessed);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -291,21 +379,66 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                                                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
                                                                                .build()));
 
-        return ManualRedactionResponse.builder().manualAddResponses(responseList).build();
+        return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build();
+    }
+
+
+    @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
+    public ManualRedactionResponse recategorizeBulkLocal(String dossierId, String fileId, RecategorizationBulkLocalRequestModel recategorizationRequest) {
+
+        verifyAccess(dossierId, fileId);
+        verifyRequest(recategorizationRequest.isRectangle(), recategorizationRequest.getPosition(), recategorizationRequest.getValue());
+        Set<RecategorizationRequestModel> recategorizationRequestModels;
+        FileModel status = fileStatusService.getStatus(fileId);
+
+        Set<EntityLogEntry> entries;
+        if (!status.isExcludedFromAutomaticAnalysis()) {
+            entries = getFilteredEntityLogEntries(dossierId,
+                                                  fileId,
+                                                  recategorizationRequest.isRectangle(),
+                                                  recategorizationRequest.getValue(),
+                                                  recategorizationRequest.isCaseSensitive(),
+                                                  recategorizationRequest.getOriginTypes(),
+                                                  recategorizationRequest.getOriginLegalBases(),
+                                                  recategorizationRequest.getPageNumbers(),
+                                                  recategorizationRequest.getPosition());
+
+        } else {
+
+            entries = new HashSet<>(mapper.fromLogEntryResponses(getFilteredEntityLogResponses(dossierId,
+                                                                                               fileId,
+                                                                                               recategorizationRequest.isRectangle(),
+                                                                                               recategorizationRequest.getValue(),
+                                                                                               recategorizationRequest.isCaseSensitive(),
+                                                                                               recategorizationRequest.getOriginTypes(),
+                                                                                               recategorizationRequest.getOriginLegalBases(),
+                                                                                               recategorizationRequest.getPageNumbers(),
+                                                                                               recategorizationRequest.getPosition())));
+        }
+
+        recategorizationRequestModels = entries.stream()
+                .map(entry -> RecategorizationRequestModel.builder()
+                        .annotationId(entry.getId())
+                        .type(recategorizationRequest.isRectangle() ? entry.getType() : recategorizationRequest.getType())
+                        .legalBasis(recategorizationRequest.getLegalBasis())
+                        .section(recategorizationRequest.getSection())
+                        .value(recategorizationRequest.isRectangle() ? recategorizationRequest.getValue() : entry.getValue())
+                        .comment(recategorizationRequest.getComment())
+                        .build())
+                .collect(Collectors.toSet());
+
+        return recategorizeBulk(dossierId, fileId, recategorizationRequestModels);
     }
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse resizeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                        @PathVariable(FILE_ID) String fileId,
-                                                       @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests,
-                                                       @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                       @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests) {
 
-        accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
-        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
-        accessControlService.verifyUserIsMemberOrApprover(dossierId);
+        verifyAccessAndDossierExistence(dossierId, fileId);
 
-        List<ManualAddResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, includeUnprocessed);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -315,7 +448,170 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                                                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
                                                                                .build()));
 
-        return ManualRedactionResponse.builder().manualAddResponses(responseList).build();
+        return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build();
+    }
+
+
+    private Set<EntityLogEntry> getFilteredEntityLogEntries(String dossierId,
+                                                            String fileId,
+                                                            boolean rectangle,
+                                                            String value,
+                                                            boolean caseSensitive,
+                                                            Set<String> originTypes,
+                                                            Set<String> originLegalBases,
+                                                            Set<Integer> pageNumbers,
+                                                            Position position) {
+
+        Set<EntityLogEntry> entries;
+        if (!rectangle) {
+            entries = entityLogMongoService.findEntriesByValueWithFilters(dossierId, fileId, value, caseSensitive, originTypes, originLegalBases, pageNumbers);
+        } else {
+            entries = entityLogMongoService.findEntriesByMatchingFullPositionWithFilters(dossierId, fileId, position.getRectangle(), originTypes, originLegalBases, pageNumbers);
+        }
+        return entries;
+    }
+
+
+    private List<EntityLogEntryResponse> getFilteredEntityLogResponses(String dossierId,
+                                                                       String fileId,
+                                                                       boolean rectangle,
+                                                                       String value,
+                                                                       boolean caseSensitive,
+                                                                       Set<String> originTypes,
+                                                                       Set<String> originLegalBases,
+                                                                       Set<Integer> pageNumbers,
+                                                                       Position position) {
+
+        List<EntityLogEntryResponse> entityLogEntryResponses = entityLogController.getEntityLog(dossierId, fileId, Collections.emptyList(), true).getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntryResponse -> !entityLogEntryResponse.getState().equals(EntryState.PENDING))
+                .collect(Collectors.toList());
+
+        if (!rectangle) {
+            entityLogEntryResponses = filterEntriesByValueWithFilters(entityLogEntryResponses, value, caseSensitive, originTypes, originLegalBases, pageNumbers);
+        } else {
+            entityLogEntryResponses = filterEntriesByMatchingPositionWithFilters(entityLogEntryResponses, position.getRectangle(), originTypes, originLegalBases, pageNumbers);
+        }
+
+        return entityLogEntryResponses;
+    }
+
+
+    public List<EntityLogEntryResponse> filterEntriesByValueWithFilters(List<EntityLogEntryResponse> entries,
+                                                                        String value,
+                                                                        boolean caseSensitive,
+                                                                        Set<String> originTypes,
+                                                                        Set<String> originLegalBases,
+                                                                        Set<Integer> pageNumbers) {
+
+        return entries.stream()
+                .filter(entry -> !entry.getEntryType().equals(EntryType.AREA))
+                .filter(entry -> filterByValue(entry, value, caseSensitive))
+                .filter(entry -> filterByOriginType(entry, originTypes))
+                .filter(entry -> filterByLegalBases(entry, originLegalBases))
+                .filter(entry -> filterByPageNumbers(entry, pageNumbers))
+                .collect(Collectors.toList());
+    }
+
+
+    public List<EntityLogEntryResponse> filterEntriesByMatchingPositionWithFilters(List<EntityLogEntryResponse> entries,
+                                                                                   float[] rectangle,
+                                                                                   Set<String> originTypes,
+                                                                                   Set<String> originLegalBases,
+                                                                                   Set<Integer> pageNumbers) {
+
+        return entries.stream()
+                .filter(entry -> entry.getEntryType().equals(EntryType.AREA))
+                .filter(entry -> filterByRectangle(entry, rectangle))
+                .filter(entry -> filterByOriginType(entry, originTypes))
+                .filter(entry -> filterByLegalBases(entry, originLegalBases))
+                .filter(entry -> filterByPageNumbers(entry, pageNumbers))
+                .collect(Collectors.toList());
+    }
+
+
+    private boolean filterByValue(EntityLogEntryResponse entry, String value, boolean caseSensitive) {
+
+        if (caseSensitive) {
+            return entry.getValue().equals(value);
+        } else {
+            return entry.getValue().equalsIgnoreCase(value);
+        }
+    }
+
+
+    private boolean filterByOriginType(EntityLogEntryResponse entry, Set<String> originTypes) {
+
+        return originTypes == null || originTypes.isEmpty() || originTypes.contains(entry.getType());
+    }
+
+
+    private boolean filterByLegalBases(EntityLogEntryResponse entry, Set<String> originLegalBases) {
+
+        return originLegalBases == null || originLegalBases.isEmpty() || originLegalBases.contains(entry.getLegalBasis());
+    }
+
+
+    private boolean filterByPageNumbers(EntityLogEntryResponse entry, Set<Integer> pageNumbers) {
+
+        if (pageNumbers == null || pageNumbers.isEmpty()) {
+            return true;
+        }
+        return entry.getPositions()
+                .stream()
+                .anyMatch(pos -> pageNumbers.contains(pos.getPageNumber()));
+    }
+
+
+    private boolean filterByRectangle(EntityLogEntryResponse entry, float[] rectangle) {
+
+        if (rectangle == null || rectangle.length == 0) {
+            return true;
+        }
+        return entry.getPositions()
+                .stream()
+                .anyMatch(pos -> Arrays.equals(pos.getRectangle(), rectangle));
+    }
+
+
+    private void verifyRequest(boolean isRectangle, Position position, String value) {
+
+        if (isRectangle && position == null) {
+            throw new BadRequestException("Position must be set for rectangle annotations.");
+        }
+
+        if (!isRectangle && value == null) {
+            throw new BadRequestException("Value must be set.");
+        }
+
+    }
+
+
+    private void verifyAccessForDossier(String dossierId, String fileId, boolean allDossiersAffected) {
+
+        accessControlService.checkAccessPermissionsToDossier(dossierId);
+        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
+        if (allDossiersAffected) {
+            accessControlService.verifyUserIsApprover(dossierId);
+        } else {
+            accessControlService.verifyUserIsMemberOrApprover(dossierId);
+        }
+    }
+
+
+    private void verifyAccess(String dossierId, String fileId) {
+
+        accessControlService.checkAccessPermissionsToDossier(dossierId);
+        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
+        accessControlService.verifyUserIsMemberOrApprover(dossierId);
+    }
+
+
+    private void verifyAccessAndDossierExistence(String dossierId, String fileId) {
+
+        accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
+        accessControlService.verifyFileIsNotApproved(dossierId, fileId);
+        accessControlService.verifyUserIsMemberOrApprover(dossierId);
     }
 
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java

@@ -1,111 +0,0 @@
-package com.iqser.red.persistence.service.v1.external.api.impl.controller;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
-import com.iqser.red.service.persistence.service.v1.api.external.resource.MigrationStatusResource;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*;
-
-@RestController
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@RequiredArgsConstructor
-public class MigrationStatusController implements MigrationStatusResource {
-
-    SaasMigrationService saasMigrationService;
-
-    SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
-
-    FileStatusService fileStatusService;
-
-
-    public MigrationStatusResponse migrationStatus() {
-
-        int numberOfFilesToMigrate = saasMigrationStatusPersistenceService.countAll();
-
-        Map<SaasMigrationStatus, Integer> filesInStatus = new HashMap<>();
-        filesInStatus.put(MIGRATION_REQUIRED, saasMigrationStatusPersistenceService.countByStatus(MIGRATION_REQUIRED));
-        filesInStatus.put(DOCUMENT_FILES_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(DOCUMENT_FILES_MIGRATED));
-        filesInStatus.put(REDACTION_LOGS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(REDACTION_LOGS_MIGRATED));
-        filesInStatus.put(ANNOTATION_IDS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(ANNOTATION_IDS_MIGRATED));
-        filesInStatus.put(FINISHED, saasMigrationStatusPersistenceService.countByStatus(FINISHED));
-        filesInStatus.put(ERROR, saasMigrationStatusPersistenceService.countByStatus(ERROR));
-
-        var filesInErrorState = saasMigrationStatusPersistenceService.findAllByStatus(ERROR);
-
-        var errorCauses = filesInErrorState.stream()
-                .collect(Collectors.toMap(errorFile -> errorFile.getDossierId() + "/" + errorFile.getFileId(), SaasMigrationStatusEntity::getErrorCause));
-
-        return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> startMigrationForFile(String dossierId, String fileId) {
-
-        if (!fileStatusService.fileExists(fileId)) {
-            throw new NotFoundException(String.format("File with id %s does not exist", fileId));
-        }
-
-        saasMigrationService.startMigrationForFile(dossierId, fileId);
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> revertMigrationForFile(String dossierId, String fileId) {
-
-        if (!fileStatusService.fileExists(fileId)) {
-            throw new NotFoundException(String.format("File with id %s does not exist", fileId));
-        }
-
-        if (!saasMigrationStatusPersistenceService.findById(fileId).getStatus().equals(FINISHED)) {
-            throw new BadRequestException(String.format("File with id %s is not migrated yet, can't revert.", fileId));
-        }
-
-        saasMigrationService.revertMigrationForFile(dossierId, fileId);
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> requeueErrorFiles() {
-
-        MigrationStatusResponse migrationStatus = migrationStatus();
-        if (!migrationIsFinished(migrationStatus)) {
-            throw new BadRequestException("There are still files processing, please wait until migration has finished to retry!");
-        }
-
-        saasMigrationService.requeueErrorFiles();
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    private static boolean migrationIsFinished(MigrationStatusResponse migrationStatus) {
-
-        return migrationStatus.getFilesInStatus().entrySet()
-                .stream()
-                .filter(e -> e.getValue() > 0)
-                .allMatch(e -> e.getKey().equals(FINISHED) || e.getKey().equals(ERROR));
-    }
-
-}

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java

@@ -118,11 +118,12 @@ public class ReanalysisController implements ReanalysisResource {
     @PreAuthorize("hasAuthority('" + REANALYZE_FILE + "')")
     public void ocrFile(@PathVariable(DOSSIER_ID) String dossierId,
                         @PathVariable(FILE_ID) String fileId,
-                        @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force) {
+                        @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force,
+                        @RequestParam(value = ALL_PAGES, required = false, defaultValue = FALSE) boolean allPages) {
 
         accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
         validateOCR(dossierId, fileId);
-        reanalysisService.ocrFile(dossierId, fileId, force);
+        reanalysisService.ocrFile(dossierId, fileId, force, allPages);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierId)
@@ -140,7 +141,7 @@ public class ReanalysisController implements ReanalysisResource {
 
         accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
         fileIds.forEach(fileId -> validateOCR(dossierId, fileId));
-        reanalysisService.ocrFiles(dossierId, fileIds);
+        reanalysisService.ocrFiles(dossierId, fileIds, false);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierId)

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java

@@ -1,156 +0,0 @@
-package com.iqser.red.persistence.service.v1.external.api.impl.controller;
-
-import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_REDACTION_LOG;
-import static com.iqser.red.service.persistence.management.v1.processor.service.FeignExceptionHandler.processFeignException;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.security.access.prepost.PreAuthorize;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
-import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService;
-import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
-import com.iqser.red.service.persistence.service.v1.api.external.resource.RedactionLogResource;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.FilteredRedactionLogRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-
-import feign.FeignException;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-
-@RestController
-@RequiredArgsConstructor
-@Deprecated(forRemoval = true)
-public class RedactionLogController implements RedactionLogResource {
-
-    private final RedactionLogService redactionLogService;
-    private final FileStatusService fileStatusService;
-    private final FileManagementStorageService fileManagementStorageService;
-
-
-    @Deprecated(forRemoval = true)
-    @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')")
-    public RedactionLog getRedactionLog(@PathVariable(DOSSIER_ID) String dossierId,
-                                        @PathVariable(FILE_ID) String fileId,
-                                        @RequestParam(value = "excludedType", required = false) List<String> excludedTypes,
-                                        @RequestParam(value = "withManualRedactions", required = false, defaultValue = "true") boolean withManualRedactions,
-                                        @RequestParam(value = "includeFalsePositives", required = false, defaultValue = "false") boolean includeFalsePositives) {
-
-        try {
-            return redactionLogService.getRedactionLog(dossierId, fileId, excludedTypes);
-        } catch (FeignException e) {
-            throw processFeignException(e);
-        }
-    }
-
-
-    @Deprecated(forRemoval = true)
-    @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')")
-    public SectionGrid getSectionGrid(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
-
-        try {
-            return redactionLogService.getSectionGrid(dossierId, fileId);
-        } catch (FeignException e) {
-            throw processFeignException(e);
-        }
-    }
-
-
-    @Deprecated(forRemoval = true)
-    @SneakyThrows
-    @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')")
-    public ResponseEntity<?> getSectionText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
-
-        try {
-            return buildZipFileResponseEntity(fileId, dossierId, FileType.TEXT);
-        } catch (FeignException e) {
-            throw processFeignException(e);
-        }
-    }
-
-
-    private ResponseEntity<byte[]> buildZipFileResponseEntity(String fileId, String dossierId, FileType fileType) throws IOException {
-
-        HttpHeaders httpHeaders = new HttpHeaders();
-        httpHeaders.setContentType(MediaType.parseMediaType("application/zip"));
-
-        var fileStatus = fileStatusService.getStatus(fileId);
-        String filename = fileStatus.getFilename();
-        if (filename != null) {
-            var index = filename.lastIndexOf(".");
-            String prefix = filename.substring(0, index);
-            filename = prefix + ".json";
-            httpHeaders.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=utf-8''" + StringEncodingUtils.urlEncode(prefix) + ".zip");
-        }
-
-        byte[] zipBytes = getZippedBytes(dossierId, fileId, filename, fileType);
-        httpHeaders.setContentLength(zipBytes.length);
-        return new ResponseEntity<>(zipBytes, httpHeaders, HttpStatus.OK);
-    }
-
-
-    @Deprecated(forRemoval = true)
-    @PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')")
-    public RedactionLog getFilteredRedactionLog(@PathVariable(DOSSIER_ID) String dossierId,
-                                                @PathVariable(FILE_ID) String fileId,
-                                                @RequestBody FilteredRedactionLogRequest filteredRedactionLogRequest) {
-
-        try {
-
-            return redactionLogService.getFilteredRedactionLog(dossierId, fileId, filteredRedactionLogRequest);
-
-        } catch (FeignException e) {
-            throw processFeignException(e);
-        }
-
-    }
-
-
-    private byte[] getZippedBytes(String dossierId, String fileId, String filename, FileType fileType) throws IOException {
-
-        try {
-            String objectId = dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
-
-            try (var inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), objectId)) {
-                byte[] input = inputStream.readAllBytes();
-                inputStream.close();
-                return zipBytes(filename, input);
-            }
-
-        } catch (StorageObjectDoesNotExist e) {
-            throw new RuntimeException(String.format("%s is not available", fileType.name()), e);
-        }
-    }
-
-
-    public static byte[] zipBytes(String filename, byte[] input) throws IOException {
-
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        ZipOutputStream zos = new ZipOutputStream(baos);
-        ZipEntry entry = new ZipEntry(filename);
-        entry.setSize(input.length);
-        zos.putNextEntry(entry);
-        zos.write(input);
-        zos.closeEntry();
-        zos.close();
-        return baos.toByteArray();
-    }
-
-}

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RulesController.java

@@ -23,8 +23,10 @@ import org.springframework.web.multipart.MultipartFile;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.FileUploadException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.RulesValidationMapper;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.RulesResource;
@@ -51,6 +53,7 @@ public class RulesController implements RulesResource {
     private final RulesPersistenceService rulesPersistenceService;
     private final RulesValidationService rulesValidationService;
     private final AuditPersistenceService auditPersistenceService;
+    private final FileStatusPersistenceService fileStatusPersistenceService;
 
 
     @Override
@@ -74,6 +77,7 @@ public class RulesController implements RulesResource {
         }
         if (!rules.isDryRun()) {
             rulesPersistenceService.setRules(rulesUploadRequest.getRules(), rulesUploadRequest.getDossierTemplateId(), rulesUploadRequest.getRuleFileType());
+            fileStatusPersistenceService.resetErrorCounter(rules.getDossierTemplateId());
         }
 
         auditPersistenceService.audit(AuditRequest.builder()
@@ -99,8 +103,11 @@ public class RulesController implements RulesResource {
     @PreAuthorize("hasAuthority('" + READ_RULES + "')")
     public RulesResponse download(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType) {
 
-        var ruleEntity = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
-        return new RulesResponse(ruleEntity.getValue(), dossierTemplateId, ruleEntity.isTimeoutDetected());
+        var ruleEntityOptional = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
+        if (ruleEntityOptional.isEmpty()) {
+            throw new NotFoundException(String.format("No rule file of type %s found for dossierTemplateId %s", ruleFileType, dossierTemplateId));
+        }
+        return new RulesResponse(ruleEntityOptional.get().getValue(), dossierTemplateId, ruleEntityOptional.get().isTimeoutDetected());
 
     }
 
@@ -153,4 +160,12 @@ public class RulesController implements RulesResource {
         return new ResponseEntity<>(new InputStreamResource(is), httpHeaders, HttpStatus.OK);
     }
 
+
+    @Override
+    @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
+    public void unlockRules(String dossierTemplateId, RuleFileType ruleFileType) {
+
+        rulesPersistenceService.resetTimeoutDetected(dossierTemplateId, ruleFileType);
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java

@@ -14,21 +14,29 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
 import org.springframework.security.access.AccessDeniedException;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
 import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
+import com.iqser.red.service.persistence.management.v1.processor.service.ApprovalVerificationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
+import com.iqser.red.service.persistence.management.v1.processor.service.FilterByPermissionsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
@@ -38,12 +46,17 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.notification.NotificationType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.warning.ApproveResponse;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -64,6 +77,8 @@ public class StatusController implements StatusResource {
     private final AccessControlService accessControlService;
     private final NotificationPersistenceService notificationPersistenceService;
     private final DossierACLService dossierACLService;
+    private final ApprovalVerificationService approvalVerificationService;
+    private final FilterByPermissionsService filterByPermissionsService;
 
 
     @Override
@@ -79,6 +94,28 @@ public class StatusController implements StatusResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
+    public JSONPrimitive<Map<String, List<FileStatus>>> getFilesByIds(@RequestBody JSONPrimitive<Map<String, Set<String>>> filesByDossier) {
+
+        // filter dossiers by view
+        var accessibleDossierIds = filterByPermissionsService.onlyViewableDossierIds(new ArrayList<>(filesByDossier.getValue().keySet()));
+        var response = new HashMap<String, List<FileStatus>>();
+        for (var dossierId : accessibleDossierIds) {
+            var allFoundFiles = fileStatusManagementService.findAllDossierIdAndIds(dossierId,
+                                                                                   filesByDossier.getValue()
+                                                                                           .get(dossierId));
+            response.put(dossierId,
+                         allFoundFiles.stream()
+                                 .map(FileStatusMapper::toFileStatus)
+                                 .collect(Collectors.toList()));
+        }
+
+        return new JSONPrimitive<>(response);
+
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
     public Map<String, List<FileStatus>> getDossierStatus(@RequestBody List<String> dossierIds) {
@@ -299,31 +336,44 @@ public class StatusController implements StatusResource {
 
 
     @PreAuthorize("hasAuthority('" + SET_STATUS_APPROVED + "')")
-    public void setStatusApproved(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
+    public ApproveResponse setStatusApproved(@PathVariable(DOSSIER_ID) String dossierId,
+                                             @PathVariable(FILE_ID) String fileId,
+                                             @RequestParam(value = FORCE_REQUEST_PARAM, required = false, defaultValue = "false") boolean force) {
 
         accessControlService.checkAccessPermissionsToDossier(dossierId);
         accessControlService.verifyUserIsApprover(dossierId);
-        setStatusApprovedForFile(dossierId, fileId);
-        auditPersistenceService.audit(AuditRequest.builder()
-                                              .userId(KeycloakSecurity.getUserId())
-                                              .objectId(fileId)
-                                              .category(AuditCategory.DOCUMENT.name())
-                                              .message("Document status was changed to Approved")
-                                              .details(Map.of(DOSSIER_ID, dossierId))
-                                              .build());
-
-        var dossier = dossierACLService.enhanceDossierWithACLData(dossierManagementService.getDossierById(dossierId, false, false));
-        if (!dossier.getOwnerId().equals(KeycloakSecurity.getUserId())) {
-
-            var fileStatus = fileStatusManagementService.getFileStatus(fileId);
-
-            notificationPersistenceService.insertNotification(AddNotificationRequest.builder()
-                                                                      .userId(dossier.getOwnerId())
-                                                                      .issuerId(KeycloakSecurity.getUserId())
-                                                                      .notificationType(NotificationType.DOCUMENT_APPROVED.name())
-                                                                      .target(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, FILE_NAME, fileStatus.getFilename()))
-                                                                      .build());
+        ApproveResponse approveResponse = new ApproveResponse(fileId, false, new ArrayList<>());
+        if (!force) {
+            approveResponse = approvalVerificationService.verifyApprovalOfFile(dossierId, fileId);
         }
+        if (!approveResponse.isHasWarnings()) {
+            setStatusApprovedForFile(dossierId, fileId);
+            auditPersistenceService.audit(AuditRequest.builder()
+                                                  .userId(KeycloakSecurity.getUserId())
+                                                  .objectId(fileId)
+                                                  .category(AuditCategory.DOCUMENT.name())
+                                                  .message("Document status was changed to Approved")
+                                                  .details(Map.of(DOSSIER_ID, dossierId))
+                                                  .build());
+
+            var dossier = dossierACLService.enhanceDossierWithACLData(dossierManagementService.getDossierById(dossierId, false, false));
+
+            if (dossier.getOwnerId() == null) {
+                throw new ConflictException("Dossier has no owner!");
+            }
+            if (!dossier.getOwnerId().equals(KeycloakSecurity.getUserId())) {
+
+                var fileStatus = fileStatusManagementService.getFileStatus(fileId);
+
+                notificationPersistenceService.insertNotification(AddNotificationRequest.builder()
+                                                                          .userId(dossier.getOwnerId())
+                                                                          .issuerId(KeycloakSecurity.getUserId())
+                                                                          .notificationType(NotificationType.DOCUMENT_APPROVED.name())
+                                                                          .target(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, FILE_NAME, fileStatus.getFilename()))
+                                                                          .build());
+            }
+        }
+        return approveResponse;
     }
 
 
@@ -357,8 +407,10 @@ public class StatusController implements StatusResource {
 
     private void generatePossibleUnassignedFromFileNotification(String dossierId, String fileId, FileModel oldFileStatus, String newAssigneeId) {
 
-        if (oldFileStatus.getAssignee() == null || newAssigneeId == null || oldFileStatus.getAssignee().equals(newAssigneeId) || KeycloakSecurity.getUserId()
-                .equals(oldFileStatus.getAssignee())) {
+        if (oldFileStatus.getAssignee() == null
+            || newAssigneeId == null && oldFileStatus.getAssignee() == null
+            || oldFileStatus.getAssignee().equals(newAssigneeId)
+            || KeycloakSecurity.getUserId().equals(oldFileStatus.getAssignee())) {
             return;
         }
 
@@ -398,14 +450,22 @@ public class StatusController implements StatusResource {
 
     @Override
     @PreAuthorize("hasAuthority('" + SET_STATUS_APPROVED + "')")
-    public void setStatusApprovedForList(String dossierId, List<String> fileIds) {
+    public List<ApproveResponse> setStatusApprovedForList(String dossierId,
+                                                          List<String> fileIds,
+                                                          @RequestParam(value = FORCE_REQUEST_PARAM, required = false, defaultValue = "false") boolean force) {
 
+        List<ApproveResponse> approveResponses = new ArrayList<>();
         accessControlService.checkAccessPermissionsToDossier(dossierId);
         accessControlService.verifyUserIsApprover(dossierId);
 
         dossierManagementService.getDossierById(dossierId, false, false);
 
-        fileIds.forEach(fileId -> setStatusApproved(dossierId, fileId));
+        if (fileIds.size() > 50) {
+            throw new BadRequestException("Maximum amount of files that can be approved at once is 50.");
+        }
+
+        fileIds.forEach(fileId -> approveResponses.add(setStatusApproved(dossierId, fileId, force)));
+        return approveResponses;
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -18,13 +18,14 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService;
+import com.iqser.red.service.persistence.management.v1.processor.migration.MigrationController;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource;
@@ -32,6 +33,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadRes
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalyzeFilesResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
@@ -48,28 +50,31 @@ public class SupportController implements SupportResource {
     private final FileStatusManagementService fileStatusManagementService;
     private final FileExchangeExportService fileExchangeExportService;
     private final FileExchangeImportService fileExchangeImportService;
+    private final DatasetExchangeService datasetExchangeService;
 
 
     @Override
-    public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
+    public ReanalyzeFilesResponse reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
 
-        reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings);
+        return new ReanalyzeFilesResponse(reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings));
     }
 
 
     @Override
-    public void reanalyzeAllErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
+    public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
+                                               @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr) {
 
-        reanalysisService.reanalyzeAllErrorFiles(repeatStructureAnalysis);
+        reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis, runOcr);
     }
 
 
     @Override
     public void reanalyzeErrorFilesBulkForDossier(@PathVariable(DOSSIER_ID) String dossierId,
                                                   @RequestBody List<String> fileIds,
-                                                  @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
+                                                  @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
+                                                  @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr) {
 
-        reanalysisService.reanalyzeGivenErrorFilesInDossier(dossierId, new HashSet<>(fileIds), repeatStructureAnalysis);
+        reanalysisService.reanalyzeGivenErrorFilesInDossier(dossierId, new HashSet<>(fileIds), repeatStructureAnalysis, runOcr);
 
     }
 
@@ -127,6 +132,13 @@ public class SupportController implements SupportResource {
     }
 
 
+    @Override
+    public DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
+
+        return datasetExchangeService.prepareExport(dossierTemplateId);
+    }
+
+
     @Override
     public DownloadResponse exportFiles(String dossierTemplateId, FileExchangeExportRequest exportRequest) {
 
@@ -147,4 +159,18 @@ public class SupportController implements SupportResource {
         return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
     }
 
+
+    @Override
+    @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
+    public ImportResponse importDataset(MultipartFile file) {
+
+        byte[] bytes;
+        try {
+            bytes = file.getBytes();
+        } catch (IOException e) {
+            throw new BadRequestException("File could not be read and is likely corrupted.", e);
+        }
+        return datasetExchangeService.importDataset(KeycloakSecurity.getUserId(), bytes);
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java

@@ -23,12 +23,11 @@ import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
-import com.iqser.red.service.persistence.management.v1.processor.service.FileFormatValidationService;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
-import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileFormatValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
 import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
@@ -37,6 +36,7 @@ import com.iqser.red.service.persistence.service.v1.api.external.resource.Upload
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import feign.FeignException;
@@ -53,9 +53,9 @@ import lombok.extern.slf4j.Slf4j;
 @SuppressWarnings("PMD")
 public class UploadController implements UploadResource {
 
-    private static final int THRESHOLD_ENTRIES = 10000;
-    private static final int THRESHOLD_SIZE = 1000000000; // 1 GB
-    private static final double THRESHOLD_RATIO = 10;
+    private static final int THRESHOLD_ENTRIES = 10000;      // Maximum number of files allowed
+    private static final int THRESHOLD_SIZE = 1000000000;    // 1 GB total unzipped data
+    private static final double THRESHOLD_RATIO = 10;        // Max allowed compression ratio
 
     private final UploadService uploadService;
     private final ReanalysisService reanalysisService;
@@ -72,31 +72,25 @@ public class UploadController implements UploadResource {
                                    @Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis) {
 
         accessControlService.checkAccessPermissionsToDossier(dossierId);
-        if (file.getOriginalFilename() == null) {
+
+        String originalFilename = file.getOriginalFilename();
+        if (originalFilename == null) {
             throw new BadRequestException("Could not upload file, no filename provided.");
         }
 
-        var extension = getExtension(file.getOriginalFilename());
-
+        String extension = getExtension(originalFilename);
         try {
-            switch (extension) {
-                case "zip":
-                    return handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
-                case "csv":
-                    return uploadService.importCsv(dossierId, file.getBytes());
-                default:
-                    if (!fileFormatValidationService.getAllFileFormats().contains(extension)) {
-                        throw new BadRequestException("Invalid file uploaded");
-                    }
-                    if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
-                        throw new NotAllowedException("Insufficient permissions");
-                    }
-                    return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
-            }
+            return switch (extension) {
+                case "zip" -> handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
+                case "csv" -> uploadService.importCsv(dossierId, file.getBytes());
+                default -> {
+                    validateExtensionOrThrow(extension);
+                    yield uploadService.processSingleFile(dossierId, originalFilename, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
+                }
+            };
         } catch (IOException e) {
-            throw new BadRequestException(e.getMessage(), e);
+            throw new BadRequestException("Failed to process file: " + e.getMessage(), e);
         }
-
     }
 
 
@@ -111,7 +105,6 @@ public class UploadController implements UploadResource {
         accessControlService.verifyUserIsReviewerOrApprover(dossierId, fileId);
 
         try {
-
             reanalysisService.importRedactions(ByteContentDocument.builder().dossierId(dossierId).fileId(fileId).document(file.getBytes()).pages(pageInclusionRequest).build());
 
             auditPersistenceService.audit(AuditRequest.builder()
@@ -122,84 +115,116 @@ public class UploadController implements UploadResource {
                                                   .details(Map.of("dossierId", dossierId))
                                                   .build());
         } catch (IOException e) {
-            throw new BadRequestException(e.getMessage(), e);
+            throw new BadRequestException("Failed to import redactions: " + e.getMessage(), e);
         } catch (FeignException e) {
             throw processFeignException(e);
         }
     }
 
 
-    private String getExtension(String fileName) {
+    private void validateExtensionOrThrow(String extension) {
 
-        return fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase(Locale.ROOT);
+        if (!fileFormatValidationService.getAllFileFormats().contains(extension)) {
+            throw new BadRequestException("Invalid file uploaded (unrecognized extension).");
+        }
+        if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
+            throw new NotAllowedException("Insufficient permissions for this file type.");
+        }
     }
 
 
+    /**
+     * 1. Write the uploaded content to a temp ZIP file
+     * 2. Check the number of entries and reject if too big or if symlinks found
+     * 3. Unzip and process each file, while checking size and ratio.
+     */
     private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
 
-        File tempFile = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip");
-        try (var fileOutputStream = new FileOutputStream(tempFile)) {
-            IOUtils.write(fileContent, fileOutputStream);
+        File tempZip = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip");
+        try (FileOutputStream fos = new FileOutputStream(tempZip)) {
+            IOUtils.write(fileContent, fos);
         }
 
-        try {
-            checkForSymlinks(tempFile);
+        validateZipEntries(tempZip);
 
-            var zipData = unzip(tempFile, dossierId, keepManualRedactions, disableAutomaticAnalysis);
+        try {
+            ZipData zipData = processZipContents(tempZip, dossierId, keepManualRedactions, disableAutomaticAnalysis);
 
             if (zipData.csvBytes != null) {
                 try {
-                    var importResult = uploadService.importCsv(dossierId, zipData.csvBytes);
-                    zipData.fileUploadResult.getProcessedAttributes().addAll(importResult.getProcessedAttributes());
-                    zipData.fileUploadResult.getProcessedFileIds().addAll(importResult.getProcessedFileIds());
+                    FileUploadResult csvResult = uploadService.importCsv(dossierId, zipData.csvBytes);
+                    zipData.fileUploadResult.getProcessedAttributes().addAll(csvResult.getProcessedAttributes());
+                    zipData.fileUploadResult.getProcessedFileIds().addAll(csvResult.getProcessedFileIds());
                 } catch (Exception e) {
-                    log.debug("CSV file inside ZIP failed", e);
-                    // TODO return un-processed files to client
+                    log.debug("CSV file inside ZIP failed to import", e);
                 }
             } else if (zipData.fileUploadResult.getFileIds().isEmpty()) {
                 if (zipData.containedUnpermittedFiles) {
-                    throw new NotAllowedException("Zip file contains unpermitted files");
+                    throw new NotAllowedException("Zip file contains unpermitted files.");
                 } else {
-                    throw new BadRequestException("Only unsupported files in zip file");
+                    throw new BadRequestException("Only unsupported files in the ZIP.");
                 }
             }
 
             return zipData.fileUploadResult;
-
         } finally {
-            boolean isDeleted = tempFile.delete();
-            if (!isDeleted) {
-                log.warn("tempFile could not be deleted");
+
+            if (!tempZip.delete()) {
+                log.warn("Could not delete temporary ZIP file: {}", tempZip);
             }
         }
     }
 
 
-    private void checkForSymlinks(File tempFile) throws IOException {
+    private void validateZipEntries(File tempZip) throws IOException {
+
+        try (FileInputStream fis = new FileInputStream(tempZip); ZipFile zipFile = new ZipFile(fis.getChannel())) {
+
+            int count = 0;
+            var entries = zipFile.getEntries();
+            while (entries.hasMoreElements()) {
+                ZipArchiveEntry ze = entries.nextElement();
 
-        try (var fis = new FileInputStream(tempFile); var zipFile = new ZipFile(fis.getChannel())) {
-            for (var entryEnum = zipFile.getEntries(); entryEnum.hasMoreElements(); ) {
-                var ze = entryEnum.nextElement();
                 if (ze.isUnixSymlink()) {
-                    throw new BadRequestException("ZIP-files with symlinks are not allowed");
+                    throw new BadRequestException("ZIP-files with symlinks are not allowed.");
+                }
+
+                if (!ze.isDirectory() && !ze.getName().startsWith(".")) {
+                    count++;
+                    if (count > THRESHOLD_ENTRIES) {
+                        throw new BadRequestException("ZIP-Bomb detected: too many entries.");
+                    }
                 }
             }
         }
     }
 
 
-    private ZipData unzip(File tempFile, String dossierId, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
+    private ZipData processZipContents(File tempZip, String dossierId, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
 
-        var zipData = new ZipData();
+        ZipData zipData = new ZipData();
 
-        try (var fis = new FileInputStream(tempFile); var zipFile = new ZipFile(fis.getChannel())) {
+        try (FileInputStream fis = new FileInputStream(tempZip); ZipFile zipFile = new ZipFile(fis.getChannel())) {
 
-            for (var entryEnum = zipFile.getEntries(); entryEnum.hasMoreElements(); ) {
-                var ze = entryEnum.nextElement();
-                zipData.totalEntryArchive++;
+            var entries = zipFile.getEntries();
+            while (entries.hasMoreElements()) {
+                ZipArchiveEntry entry = entries.nextElement();
 
-                if (!ze.isDirectory()) {
-                    processFileZipEntry(ze, zipFile, dossierId, keepManualRedactions, zipData, disableAutomaticAnalysis);
+                if (entry.isDirectory() || entry.getName().startsWith(".")) {
+                    continue;
+                }
+
+                byte[] entryBytes = readEntryWithRatioCheck(entry, zipFile);
+                zipData.totalSizeArchive += entryBytes.length;
+                if (zipData.totalSizeArchive > THRESHOLD_SIZE) {
+                    throw new BadRequestException("ZIP-Bomb detected (exceeds total size limit).");
+                }
+
+                String extension = getExtension(entry.getName());
+                if ("csv".equalsIgnoreCase(extension)) {
+                    zipData.csvBytes = entryBytes;
+                } else {
+                    handleRegularFile(dossierId, entryBytes, extension, extractFileName(entry.getName()), zipData, keepManualRedactions, disableAutomaticAnalysis);
                 }
             }
         }
@@ -207,73 +232,70 @@ public class UploadController implements UploadResource {
     }
 
 
-    private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData, boolean disableAutomaticAnalysis) throws IOException {
+    private byte[] readEntryWithRatioCheck(ZipArchiveEntry entry, ZipFile zipFile) throws IOException {
 
-        var extension = getExtension(ze.getName());
+        long compressedSize = entry.getCompressedSize() > 0 ? entry.getCompressedSize() : 1;
+        try (var is = zipFile.getInputStream(entry); var bos = new ByteArrayOutputStream()) {
 
-        final String fileName;
-        if (ze.getName().lastIndexOf("/") >= 0) {
-            fileName = ze.getName().substring(ze.getName().lastIndexOf("/") + 1);
-        } else {
-            fileName = ze.getName();
-        }
+            byte[] buffer = new byte[4096];
+            int bytesRead;
+            int totalUncompressed = 0;
 
-        if (fileName.startsWith(".")) {
-            return;
-        }
+            while ((bytesRead = is.read(buffer)) != -1) {
+                bos.write(buffer, 0, bytesRead);
+                totalUncompressed += bytesRead;
 
-        var entryAsBytes = readCurrentZipEntry(ze, zipFile);
-        zipData.totalSizeArchive += entryAsBytes.length;
-
-        // 1. the uncompressed data size is too much for the application resource capacity
-        // 2. too many entries in the archive can lead to inode exhaustion of the file-system
-        if (zipData.totalSizeArchive > THRESHOLD_SIZE || zipData.totalEntryArchive > THRESHOLD_ENTRIES) {
-            throw new BadRequestException("ZIP-Bomb detected.");
-        }
-
-        if ("csv".equals(extension)) {
-            zipData.csvBytes = entryAsBytes;
-        } else if (fileFormatValidationService.getAllFileFormats().contains(extension)) {
-
-            if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
-                zipData.containedUnpermittedFiles = true;
-                return;
-            }
-            zipData.containedUnpermittedFiles = false;
-
-            try {
-                var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions, disableAutomaticAnalysis);
-                zipData.fileUploadResult.getFileIds().addAll(result.getFileIds());
-            } catch (Exception e) {
-                log.debug("PDF File inside ZIP failed", e);
-                // TODO return un-processed files to client
+                double ratio = (double) totalUncompressed / compressedSize;
+                if (ratio > THRESHOLD_RATIO) {
+                    throw new BadRequestException("ZIP-Bomb detected (compression ratio too high).");
+                }
             }
+            return bos.toByteArray();
         }
     }
 
 
-    private byte[] readCurrentZipEntry(ZipArchiveEntry ze, ZipFile zipFile) throws IOException {
+    private void handleRegularFile(String dossierId,
+                                   byte[] fileBytes,
+                                   String extension,
+                                   String fileName,
+                                   ZipData zipData,
+                                   boolean keepManualRedactions,
+                                   boolean disableAutomaticAnalysis) {
 
-        var bos = new ByteArrayOutputStream();
-
-        try (var entryStream = zipFile.getInputStream(ze)) {
-            var buffer = new byte[2048];
-            var nBytes = 0;
-            int totalSizeEntry = 0;
-
-            while ((nBytes = entryStream.read(buffer)) > 0) {
-                bos.write(buffer, 0, nBytes);
-                totalSizeEntry += nBytes;
-
-                double compressionRatio = (float) totalSizeEntry / ze.getCompressedSize();
-                if (compressionRatio > THRESHOLD_RATIO) {
-                    // ratio between compressed and uncompressed data is highly suspicious, looks like a Zip Bomb Attack
-                    throw new BadRequestException("ZIP-Bomb detected.");
-                }
-            }
+        if (!fileFormatValidationService.getAllFileFormats().contains(extension)) {
+            zipData.containedUnpermittedFiles = false;
+            return;
         }
 
-        return bos.toByteArray();
+        if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
+            zipData.containedUnpermittedFiles = true;
+            return;
+        }
+
+        try {
+            FileUploadResult result = uploadService.processSingleFile(dossierId, fileName, fileBytes, keepManualRedactions, disableAutomaticAnalysis);
+            zipData.fileUploadResult.getFileIds().addAll(result.getFileIds());
+        } catch (Exception e) {
+            log.debug("Failed to process file '{}' in ZIP: {}", fileName, e.getMessage(), e);
+        }
+    }
+
+
+    private String extractFileName(String path) {
+
+        int idx = path.lastIndexOf('/');
+        return (idx >= 0) ? path.substring(idx + 1) : path;
+    }
+
+
+    private String getExtension(String fileName) {
+
+        int idx = fileName.lastIndexOf('.');
+        if (idx < 0) {
+            return "";
+        }
+        return fileName.substring(idx + 1).toLowerCase(Locale.ROOT);
     }
 
 
@@ -282,7 +304,6 @@ public class UploadController implements UploadResource {
 
         byte[] csvBytes;
         int totalSizeArchive;
-        int totalEntryArchive;
         FileUploadResult fileUploadResult = new FileUploadResult();
         boolean containedUnpermittedFiles;
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UserStatsController.java

@@ -0,0 +1,60 @@
+package com.iqser.red.persistence.service.v1.external.api.impl.controller;
+
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_USER_STATS;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
+import com.iqser.red.service.persistence.service.v1.api.external.resource.UserStatsResource;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.UserStats;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RestController
+@RequiredArgsConstructor
+public class UserStatsController implements UserStatsResource {
+
+    private final UserService userService;
+    private final DossierService dossierService;
+    private final FileStatusPersistenceService fileStatusPersistenceService;
+    private final DossierACLService dossierACLService;
+
+
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_USER_STATS + "')")
+    public ResponseEntity<UserStats> getUserStats(String userId) {
+
+        if (userService.getUserById(userId).isEmpty()) {
+            throw new NotFoundException(String.format("The user with id %s is not found.", userId));
+        }
+        List<String> dossierMemberships = new ArrayList<>();
+        List<String> dossierOwnerships = new ArrayList<>();
+        dossierService.getAllDossiers()
+                .stream()
+                .filter(dossierEntity -> dossierEntity.getHardDeletedTime() == null)
+                .forEach(d -> {
+                    if (dossierACLService.getMembers(d.getId()).contains(userId)) {
+                        dossierMemberships.add(d.getId());
+                    }
+                    if (dossierACLService.getOwners(d.getId()).contains(userId)) {
+                        dossierOwnerships.add(d.getId());
+                    }
+                });
+
+        return new ResponseEntity<>(new UserStats(dossierMemberships.size(), dossierOwnerships.size(), this.fileStatusPersistenceService.getNumberOfAssignedFiles(userId)),
+                                    HttpStatus.OK);
+    }
+
+}

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/VersionsController.java

@@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.VersionsResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -27,6 +28,7 @@ public class VersionsController implements VersionsResource {
     private final DictionaryPersistenceService dictionaryPersistenceService;
     private final RulesPersistenceService rulesPersistenceService;
     private final AccessControlService accessControlService;
+    private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
 
 
     @Override
@@ -35,7 +37,9 @@ public class VersionsController implements VersionsResource {
 
         var result = new HashMap<String, VersionsResponse>();
         dossierTemplateIds.forEach(rsId -> {
-            VersionsResponse response = new VersionsResponse(dictionaryPersistenceService.getVersion(rsId), rulesPersistenceService.getVersion(rsId, RuleFileType.ENTITY));
+            VersionsResponse response = new VersionsResponse(dictionaryPersistenceService.getVersion(rsId),
+                                                             rulesPersistenceService.getVersion(rsId, RuleFileType.ENTITY),
+                                                             legalBasisMappingPersistenceService.getVersion(rsId));
             result.put(rsId, response);
         });
 

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java

@@ -5,25 +5,45 @@ import static com.iqser.red.service.persistence.service.v2.api.external.resource
 import static com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource.DOSSIER_TEMPLATE_ID_PARAM;
 import static com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource.FILE_ID_PARAM;
 
+import java.util.ArrayList;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.security.access.AccessDeniedException;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierController;
 import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController;
 import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMapper;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
+import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
+import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
+import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.iqser.red.service.persistence.service.v2.api.external.model.BulkComponentsRequest;
 import com.iqser.red.service.persistence.service.v2.api.external.model.Component;
 import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentOverrideList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponentsList;
 import com.iqser.red.service.persistence.service.v2.api.external.resource.ComponentResource;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
+import com.knecon.fforesight.tenantcommons.TenantProvider;
 
+import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -32,14 +52,21 @@ import lombok.experimental.FieldDefaults;
 @RestController
 @RequiredArgsConstructor
 @Tag(name = "4. Component endpoints", description = "Provides operations related to components")
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ComponentControllerV2 implements ComponentResource {
 
-    ComponentLogService componentLogService;
-    StatusController statusController;
-    FileStatusService fileStatusService;
-    DossierTemplatePersistenceService dossierTemplatePersistenceService;
-    ComponentMapper componentMapper = ComponentMapper.INSTANCE;
+    private final AccessControlService accessControlService;
+    private final ComponentLogService componentLogService;
+    private final UserService userService;
+    private final StatusController statusController;
+    private final FileStatusService fileStatusService;
+    private final DossierController dossierController;
+    private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
+    private final CurrentApplicationTypeProvider currentApplicationTypeProvider;
+    private final ComponentMapper componentMapper = ComponentMapper.INSTANCE;
+
+
+    @Value("${documine.components.filesLimit:100}")
+    private int documineComponentsFilesLimit = 100;
 
 
     @Override
@@ -48,27 +75,67 @@ public class ComponentControllerV2 implements ComponentResource {
                                         @PathVariable(FILE_ID_PARAM) String fileId,
                                         @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) {
 
+        checkApplicationType();
+        validateUserRoles(KeycloakSecurity.getUserId());
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
-        componentLogService.validateUserRoles(KeycloakSecurity.getUserId());
-        var componentLog = componentLogService.getComponentLog(dossierId, fileId, true);
+        var componentLog = componentLogService.getComponentLog(dossierId, fileId);
 
         return componentMapper.toFileComponents(componentLog, dossierTemplateId, dossierId, fileId, fileStatusService.getFileName(fileId), includeDetails);
     }
 
 
+    private void validateUserRoles(String userId) {
+
+        Optional<User> userOptional = userService.getUserById(userId);
+        if (userOptional.isPresent()) {
+            if (userOptional.get().getRoles()
+                    .stream()
+                    .noneMatch(ApplicationRoles.VALID_MEMBER_ROLES::contains)) {
+                throw new NotAllowedException("User doesn't have appropriate roles");
+            }
+        }
+    }
+
+
     @Override
     public FileComponentsList getComponentsOfDossier(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
                                                      @PathVariable(DOSSIER_ID_PARAM) String dossierId,
                                                      @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) {
 
+        checkApplicationType();
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
         var dossierFiles = statusController.getDossierStatus(dossierId);
+
+        if(dossierFiles.size() > documineComponentsFilesLimit) {
+            throw new BadRequestException(String.format("The dossier you requested components for contains %s files this is above the limit of %s files for this endpoint, please use the POST %s", dossierFiles.size(), documineComponentsFilesLimit, FILE_PATH + BULK_COMPONENTS_PATH));
+        }
+
         return new FileComponentsList(dossierFiles.stream()
                                               .map(file -> getComponents(dossierTemplateId, dossierId, file.getFileId(), includeDetails))
                                               .toList());
     }
 
 
+    @Override
+    public FileComponentsList getComponentsForFiles(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
+                                             @PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                                             @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails,
+                                             @RequestBody BulkComponentsRequest bulkComponentsRequest){
+
+        if(bulkComponentsRequest.getFileIds().size() > documineComponentsFilesLimit) {
+            throw new BadRequestException(String.format("You requested components for %s files this is above the limit of %s files for this endpoint, lower the fileIds in the request", bulkComponentsRequest.getFileIds().size(), documineComponentsFilesLimit));
+        }
+
+        checkApplicationType();
+        dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
+        dossierController.getDossier(dossierId, false, false);
+        return new FileComponentsList(bulkComponentsRequest.getFileIds().stream()
+                                              .map(fileId -> getComponents(dossierTemplateId, dossierId, fileId, includeDetails))
+                                              .toList());
+
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + GET_RSS + "')")
     public void addOverride(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@@ -76,9 +143,11 @@ public class ComponentControllerV2 implements ComponentResource {
                             @PathVariable(FILE_ID_PARAM) String fileId,
                             @RequestBody Component override) {
 
+        checkApplicationType();
+        accessControlService.verifyUserIsReviewer(dossierId, fileId);
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
-        componentLogService.addOverride(dossierId, fileId, componentMapper.toComponentLogEntry(override));
+        componentLogService.overrideComponent(dossierId, fileId, componentMapper.toComponentLogEntry(override));
     }
 
 
@@ -88,12 +157,23 @@ public class ComponentControllerV2 implements ComponentResource {
                                               @PathVariable(DOSSIER_ID_PARAM) String dossierId,
                                               @PathVariable(FILE_ID_PARAM) String fileId) {
 
+        checkApplicationType();
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
-        var overrides = componentLogService.getOverrides(dossierId, fileId);
+        FileModel status = fileStatusService.getStatus(fileId);
+
+        var overrides = componentLogService.getComponentLog(dossierId, fileId).getComponentLogEntries()
+                .stream()
+                .filter(ComponentLogEntry::isOverridden)
+                .collect(Collectors.toList());
         var componentOverrides = componentMapper.toComponents(overrides);
 
-        return ComponentOverrideList.builder().dossierTemplateId(dossierTemplateId).dossierId(dossierId).fileId(fileId).componentOverrides(componentOverrides).build();
+        return ComponentOverrideList.builder()
+                .dossierTemplateId(dossierTemplateId)
+                .dossierId(dossierId)
+                .fileId(fileId)
+                .componentOverrides(status.isSoftOrHardDeleted() ? new ArrayList<>() : componentOverrides)
+                .build();
     }
 
 
@@ -104,9 +184,19 @@ public class ComponentControllerV2 implements ComponentResource {
                                 @PathVariable(FILE_ID_PARAM) String fileId,
                                 @RequestBody RevertOverrideRequest revertOverrideRequest) {
 
+        checkApplicationType();
+        accessControlService.verifyUserIsReviewer(dossierId, fileId);
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
         componentLogService.revertOverrides(dossierId, fileId, revertOverrideRequest);
     }
 
+
+    private void checkApplicationType() {
+
+        if (!currentApplicationTypeProvider.isDocuMine()) {
+            throw new NotAllowedException("Components can only be accessed in DocuMine");
+        }
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierControllerV2.java

@@ -21,6 +21,7 @@ import com.iqser.red.persistence.service.v1.external.api.impl.controller.Downloa
 import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierAttributesManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
@@ -42,24 +43,25 @@ import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 
 @RestController
 @RequiredArgsConstructor
 @Tag(name = "2. Dossier endpoints", description = "Provides operations related to dossiers")
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DossierControllerV2 implements DossierResource {
 
-    private final DossierTemplateController dossierTemplateController;
-    private final DossierController dossierController;
-    private final AccessControlService accessControlService;
-    private final DossierAttributesManagementService dossierAttributesManagementService;
-    private final AuditPersistenceService auditPersistenceService;
-    private final DownloadController downloadController;
-    private final StatusController statusController;
-    private final DownloadStatusPersistenceService downloadStatusPersistenceService;
-
-    @Value("${application.type}")
-    private String applicationType;
+    DossierTemplateController dossierTemplateController;
+    DossierController dossierController;
+    AccessControlService accessControlService;
+    DossierAttributesManagementService dossierAttributesManagementService;
+    AuditPersistenceService auditPersistenceService;
+    DownloadController downloadController;
+    StatusController statusController;
+    DownloadStatusPersistenceService downloadStatusPersistenceService;
+    CurrentApplicationTypeProvider currentApplicationTypeProvider;
 
 
     public DossierList getDossiers(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@@ -191,7 +193,7 @@ public class DossierControllerV2 implements DossierResource {
                 .description(dossier.getDescription())
                 .ownerId(dossier.getOwnerId())
                 .memberIds(dossier.getMemberIds())
-                .approverIds(applicationType.equals("DocuMine") ? dossier.getMemberIds() : dossier.getApproverIds()) // for DocuMine, the members are always set as approvers
+                .approverIds(currentApplicationTypeProvider.isDocuMine() ? dossier.getMemberIds() : dossier.getApproverIds()) // for DocuMine, the members are always set as approvers
                 .downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
                 .reportTemplateIds(dossier.getReportTemplateIds())
                 .watermarkId(null)

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java

@@ -1,10 +1,12 @@
 package com.iqser.red.persistence.service.v2.external.api.impl.controller;
 
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_REPORT_TEMPLATES;
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_DATA_FORMATS;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_DOSSIER_ATTRIBUTES_CONFIG;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_DOSSIER_STATUS;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_FILE_ATTRIBUTES_CONFIG;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_RULES;
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.WRITE_DATA_FORMATS;
 import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.WRITE_RULES;
 import static java.lang.String.format;
 
@@ -18,6 +20,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 
 import org.springframework.core.io.InputStreamResource;
 import org.springframework.http.HttpHeaders;
@@ -35,14 +38,18 @@ import com.google.common.base.Strings;
 import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierTemplateController;
 import com.iqser.red.persistence.service.v1.external.api.impl.controller.FileAttributesController;
 import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMappingMapper;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DateFormatsEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ComponentDefinitionEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentDefinitionService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DateFormatsValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
@@ -58,6 +65,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.component.C
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionUpdateRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DateFormatPatternErrorMessage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest;
 import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel;
@@ -89,10 +97,13 @@ import lombok.experimental.FieldDefaults;
 public class DossierTemplateControllerV2 implements DossierTemplateResource {
 
     private static final String RULES_DOWNLOAD_FILE_NAME_SUFFIX = "-rules.drl";
+    private static final String DATE_FORMAT_FILE_NAME = "date_formats.txt";
 
     DossierTemplateController dossierTemplateController;
     RulesPersistenceService rulesPersistenceService;
+    DateFormatsPersistenceService dateFormatsPersistenceService;
     RulesValidationService rulesValidationService;
+    DateFormatsValidationService dateFormatsValidationService;
     AuditPersistenceService auditPersistenceService;
     FileAttributesController fileAttributesController;
     ComponentMappingService componentMappingService;
@@ -156,6 +167,59 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     }
 
 
+    @SneakyThrows
+    @PreAuthorize("hasAuthority('" + WRITE_DATA_FORMATS + "')")
+    public ResponseEntity<?> uploadDateFormats(String dossierTemplateId, MultipartFile file) {
+
+        dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
+
+        String originalFilename = file.getOriginalFilename();
+        if (originalFilename == null || !originalFilename.endsWith(".txt")) {
+            throw new BadRequestException("Only .txt files can be parsed.");
+        }
+
+        String dateFormats = new String(file.getBytes(), StandardCharsets.UTF_8);
+
+        List<DateFormatPatternErrorMessage> dateFormatPatternErrorMessages = dateFormatsValidationService.validateDateFormats(dateFormats);
+        if (!dateFormatPatternErrorMessages.isEmpty()) {
+            return new ResponseEntity<>(dateFormatPatternErrorMessages, HttpStatus.UNPROCESSABLE_ENTITY);
+        }
+
+        dateFormatsPersistenceService.setDateFormats(dateFormats, dossierTemplateId);
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(dossierTemplateId)
+                                              .category(AuditCategory.DOSSIER_TEMPLATE.name())
+                                              .message("date formats have been updated")
+                                              .build());
+
+        return new ResponseEntity<>(HttpStatus.NO_CONTENT);
+    }
+
+
+    @SneakyThrows
+    @PreAuthorize("hasAuthority('" + READ_DATA_FORMATS + "')")
+    public ResponseEntity<?> downloadDateFormats(String dossierTemplateId) {
+
+        dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
+
+        Optional<DateFormatsEntity> dateFormatsOptional = dateFormatsPersistenceService.getDateFormats(dossierTemplateId);
+        if (dateFormatsOptional.isEmpty()) {
+            throw new NotFoundException(String.format("No date formats found for dossierTemplateId %s", dossierTemplateId));
+        }
+        var data = dateFormatsOptional.get().getValue().getBytes(StandardCharsets.UTF_8);
+
+        HttpHeaders httpHeaders = new HttpHeaders();
+        httpHeaders.setContentType(MediaType.TEXT_PLAIN);
+
+        httpHeaders.add("Content-Disposition", "attachment" + "; filename*=utf-8''" + StringEncodingUtils.urlEncode(DATE_FORMAT_FILE_NAME));
+
+        InputStream is = new ByteArrayInputStream(data);
+
+        return new ResponseEntity<>(new InputStreamResource(is), httpHeaders, HttpStatus.OK);
+    }
+
+
     @PreAuthorize("hasAuthority('" + READ_FILE_ATTRIBUTES_CONFIG + "')")
     public FileAttributeDefinitionList getFileAttributeDefinitions(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) {
 
@@ -183,6 +247,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
                                                  .filterable(fileAttributeConfig.isFilterable())
                                                  .displayedInFileList(fileAttributeConfig.isDisplayedInFileList())
                                                  .build())
+                        .includeInCsvExport(fileAttributeConfig.isIncludeInCsvExport())
                         .build())
                 .toList();
 
@@ -205,7 +270,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     @Override
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
-    public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter) {
+    public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter, String quoteChar) {
 
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
@@ -221,18 +286,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
             throw new BadRequestException(format("The provided file name \"%s\" is not valid!", nameToUse));
         }
 
-        if (Strings.isNullOrEmpty(delimiter)) {
-            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
-        } else if (delimiter.length() != 1) {
-            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
-        }
-        char cleanDelimiter = delimiter.charAt(0);
+        char cleanDelimiter = getDelimiter(delimiter);
+        char cleanQuoteChar = getQuoteChar(quoteChar);
 
         Path mappingFile = saveToFile(file);
 
         try {
 
-            ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, cleanDelimiter, encoding, mappingFile.toFile());
+            ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
+                                                                               nameToUse,
+                                                                               fileName,
+                                                                               cleanDelimiter,
+                                                                               encoding,
+                                                                               mappingFile.toFile(),
+                                                                               cleanQuoteChar);
 
             return componentMappingMapper.toModel(metaData);
         } finally {
@@ -245,19 +312,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     @Override
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
-    public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String name, String encoding, String delimiter) {
+    public ComponentMappingMetadataModel updateMapping(String dossierTemplateId,
+                                                       String componentMappingId,
+                                                       MultipartFile file,
+                                                       String name,
+                                                       String encoding,
+                                                       String delimiter,
+                                                       String quoteChar) {
 
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
         String nameToUse = validateFileName(file, name);
 
-        if (Strings.isNullOrEmpty(delimiter)) {
-            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
-        } else if (delimiter.length() != 1) {
-            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
-        }
-        char cleanDelimiter = delimiter.charAt(0);
-
+        char cleanDelimiter = getDelimiter(delimiter);
+        char cleanQuoteChar = getQuoteChar(quoteChar);
 
         Path mappingFile = saveToFile(file);
 
@@ -268,7 +336,8 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
                                                                                      encoding,
                                                                                      cleanDelimiter,
                                                                                      mappingFile.toFile(),
-                                                                                     file.getOriginalFilename());
+                                                                                     file.getOriginalFilename(),
+                                                                                     cleanQuoteChar);
 
             return componentMappingMapper.toModel(resultMetaData);
         } finally {
@@ -277,6 +346,28 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     }
 
 
+    private static char getDelimiter(String delimiter) {
+
+        if (Strings.isNullOrEmpty(delimiter)) {
+            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
+        } else if (delimiter.length() != 1) {
+            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
+        }
+        return delimiter.charAt(0);
+    }
+
+
+    private static char getQuoteChar(String quoteChar) {
+
+        if (Strings.isNullOrEmpty(quoteChar)) {
+            throw new BadRequestException("The provided quoteChar is not valid! Can't be null or empty.");
+        } else if (quoteChar.length() != 1) {
+            throw new BadRequestException(format("The provided quoteChar %s is not valid! Only a single character is allowed.", quoteChar));
+        }
+        return quoteChar.charAt(0);
+    }
+
+
     private static String validateFileName(MultipartFile file, String name) {
 
         if (Strings.isNullOrEmpty(file.getOriginalFilename()) || !file.getOriginalFilename().endsWith(".csv")) {
@@ -359,11 +450,15 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
         return new DossierAttributeDefinitionList(dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId)
                                                           .stream()
                                                           .map(config -> DossierAttributeDefinition.builder()
-
                                                                   .id(config.getId())
                                                                   .name(config.getLabel())
                                                                   .type(config.getType())
                                                                   .reportingPlaceholder(config.getPlaceholder())
+                                                                  .displaySettings(DossierAttributeDefinition.DossierDisplaySettings.builder()
+                                                                                           .editable(config.isEditable())
+                                                                                           .filterable(config.isFilterable())
+                                                                                           .displayedInDossierList(config.isDisplayedInDossierList())
+                                                                                           .build())
                                                                   .build())
                                                           .toList());
     }
@@ -502,9 +597,11 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
 
     private ResponseEntity<?> downloadRules(String dossierTemplateId, RuleFileType ruleFileType) {
 
-        RuleSetEntity ruleEntity = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
-
-        var data = ruleEntity.getValue().getBytes(StandardCharsets.UTF_8);
+        Optional<RuleSetEntity> ruleEntityOptional = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
+        if (ruleEntityOptional.isEmpty()) {
+            throw new NotFoundException(String.format("No rule file of type %s found for dossierTemplateId %s", ruleFileType, dossierTemplateId));
+        }
+        var data = ruleEntityOptional.get().getValue().getBytes(StandardCharsets.UTF_8);
 
         HttpHeaders httpHeaders = new HttpHeaders();
         httpHeaders.setContentType(MediaType.TEXT_PLAIN);

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DownloadControllerV2.java

@@ -1,6 +1,7 @@
 package com.iqser.red.persistence.service.v2.external.api.impl.controller;
 
 import java.util.List;
+import java.util.Optional;
 
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RestController;
@@ -8,7 +9,11 @@ import org.springframework.web.bind.annotation.RestController;
 import com.iqser.red.persistence.service.v1.external.api.impl.controller.DownloadController;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
+import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
+import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RemoveDownloadRequest;
 import com.iqser.red.service.persistence.service.v2.api.external.model.DownloadStatus;
 import com.iqser.red.service.persistence.service.v2.api.external.model.DownloadStatusList;
@@ -26,10 +31,13 @@ public class DownloadControllerV2 implements DownloadResource {
 
     private final DownloadController downloadController;
     private final DownloadStatusPersistenceService downloadStatusPersistenceService;
+    private final UserService userService;
+
 
     @Transactional
     public DownloadStatusList getDownloadStatusList() {
 
+        validateUserRoles(KeycloakSecurity.getUserId());
         var downloads = downloadStatusPersistenceService.getStatusesByUser(KeycloakSecurity.getUserId());
 
         return new DownloadStatusList(downloads.stream().map(
@@ -61,6 +69,7 @@ public class DownloadControllerV2 implements DownloadResource {
     @Transactional
     public DownloadStatus getDownloadStatus(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) {
 
+        validateUserRoles(KeycloakSecurity.getUserId());
         var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId);
 
         return DownloadStatus.builder()
@@ -88,6 +97,7 @@ public class DownloadControllerV2 implements DownloadResource {
 
     public void deleteDownload(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) {
 
+        validateUserRoles(KeycloakSecurity.getUserId());
         var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId);
         downloadController.deleteDownloadStatus(new RemoveDownloadRequest(List.of(status.getStorageId())));
     }
@@ -95,8 +105,22 @@ public class DownloadControllerV2 implements DownloadResource {
 
     public void download(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) {
 
+        validateUserRoles(KeycloakSecurity.getUserId());
         var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId);
         downloadController.downloadFile(status.getStorageId());
     }
 
+
+    private void validateUserRoles(String userId) {
+
+        Optional<User> userOptional = userService.getUserById(userId);
+        if (userOptional.isPresent()) {
+            if (userOptional.get().getRoles()
+                    .stream()
+                    .noneMatch(ApplicationRoles.VALID_MEMBER_ROLES::contains)) {
+                throw new NotAllowedException("User doesn't have appropriate roles");
+            }
+        }
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/FileControllerV2.java

@@ -156,8 +156,6 @@ public class FileControllerV2 implements FileResource {
                                               @PathVariable(FILE_ID_PARAM) String fileId,
                                               @RequestBody DownloadRequest downloadRequest) {
 
-        fileStatusManagementService.getFileStatus(fileId, false);
-
         return prepareBulkDownload(dossierTemplateId,
                                    dossierId,
                                    BulkDownloadRequest.builder()

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/mapper/ComponentMapper.java

@@ -21,11 +21,11 @@ public interface ComponentMapper {
     ComponentMapper INSTANCE = Mappers.getMapper(ComponentMapper.class);
 
 
-    @Mapping(source = "componentLogEntityReferences", target = "entityReferences")
+    @Mapping(target = "entityReferences", source = "componentLogEntityReferences")
     ComponentValue toComponentValue(ComponentLogEntryValue entry);
 
 
-    @Mapping(source = "entityReferences", target = "componentLogEntityReferences")
+    @Mapping(target = "componentLogEntityReferences", source = "entityReferences")
     ComponentLogEntryValue toComponentLogEntry(ComponentValue value);
 
 
@@ -34,11 +34,13 @@ public interface ComponentMapper {
 
     List<ComponentLogEntryValue> toComponentLogEntries(List<ComponentValue> values);
 
+    @Mapping(target = "componentValues", source = "values")
     Component toComponent(ComponentLogEntry entry);
 
 
     List<Component> toComponents(List<ComponentLogEntry> entries);
 
+    @Mapping(target = "values", source = "componentValues")
     ComponentLogEntry toComponentLogEntry(Component component);
 
 
@@ -47,7 +49,7 @@ public interface ComponentMapper {
         Map<String, List<String>> basicComponent = new LinkedHashMap<>();
         for (ComponentLogEntry componentLogEntry : componentLog.getComponentLogEntries()) {
             basicComponent.put(componentLogEntry.getName(),
-                               componentLogEntry.getComponentValues()
+                               componentLogEntry.getValues()
                                        .stream()
                                        .map(ComponentLogEntryValue::getValue)
                                        .toList());

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ComponentLogResource.java

@@ -1,75 +0,0 @@
-package com.iqser.red.service.persistence.service.v1.api.external.resource;
-
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
-import org.springframework.web.bind.annotation.ResponseStatus;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentsOverrides;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
-
-@ResponseStatus(value = HttpStatus.OK)
-public interface ComponentLogResource {
-
-    String COMPONENT_LOG_PATH = ExternalApi.BASE_PATH + "/componentLog";
-
-    String OVERRIDE_PATH = "/override";
-
-    String DOSSIER_TEMPLATE_ID = "dossierTemplateId";
-    String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID + "}";
-
-    String DOSSIER_ID = "dossierId";
-    String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
-
-    String FILE_ID = "fileId";
-    String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
-
-
-    @GetMapping(value = COMPONENT_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Gets the component log for a fileId", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The component log is not found.")})
-    ComponentLog getComponentLog(@PathVariable(DOSSIER_ID) String dossierId,
-                                 @PathVariable(FILE_ID) String fileId,
-                                 @RequestParam(name = "includeOverrides", defaultValue = "true") boolean includeOverrides);
-
-
-    @ResponseBody
-    @ResponseStatus(value = HttpStatus.NO_CONTENT)
-    @PostMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Adds overrides for components", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void addOverride(@PathVariable(DOSSIER_ID) String dossierId,
-                     @PathVariable(FILE_ID) String fileId,
-                     @RequestBody ComponentsOverrides componentsOverrides);
-
-
-    @ResponseBody
-    @ResponseStatus(value = HttpStatus.OK)
-    @GetMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Gets overrides for components", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found")})
-    ComponentsOverrides getOverrides(@PathVariable(DOSSIER_ID) String dossierId,
-                                     @PathVariable(FILE_ID) String fileId);
-
-
-    @ResponseBody
-    @ResponseStatus(value = HttpStatus.NO_CONTENT)
-    @PostMapping(value = COMPONENT_LOG_PATH + OVERRIDE_PATH + "/revert" + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Reverts overrides for components", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void revertOverrides(@PathVariable(DOSSIER_ID) String dossierId,
-                         @PathVariable(FILE_ID) String fileId,
-                         @RequestBody RevertOverrideRequest revertOverrideRequest);
-
-}

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java

@@ -223,10 +223,9 @@ public interface DictionaryResource {
     void changeFlags(@PathVariable(TYPE_PARAMETER_NAME) String type,
                      @PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
                      @PathVariable(DOSSIER_ID_PARAMETER_NAME) String dossierId,
-                     @RequestParam(value = "addToDictionary") boolean addToDictionary);
+                     @RequestParam(value = "addToDictionaryAction") boolean addToDictionaryAction);
 
 
-    @ResponseStatus(HttpStatus.ACCEPTED)
     @PostMapping(value = DICTIONARY_REST_PATH + DIFFERENCE + DOSSIER_TEMPLATE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Returns the difference between the dictionaries of the dossier template and all the dossiers inside the template for a list of given types.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Successfully returned DictionaryDifferenceResponse."), @ApiResponse(responseCode = "400", description = "The request is not valid.")})

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DossierResource.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.external.resource;
 
 import java.time.OffsetDateTime;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.springframework.http.HttpStatus;
@@ -17,6 +18,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierInformation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
@@ -29,10 +31,12 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
 public interface DossierResource {
 
     String DOSSIER_REST_PATH = ExternalApi.BASE_PATH + "/dossier";
+    String BY_ID_PATH = "/by-id";
     String DOSSIER_TEMPLATE_PATH = "/dossier-template";
     String DOSSIER_INFO_PATH = "/info";
     String DELETED_DOSSIERS_PATH = ExternalApi.BASE_PATH + "/deleted-dossiers";
     String CHANGES_DETAILS_PATH = "/changes/details";
+    String CHANGES_DETAILS_V2_PATH = "/changes/details/v2";
     String HARD_DELETE_PATH = "/hard-delete";
     String UNDELETE_PATH = "/restore";
 
@@ -62,6 +66,12 @@ public interface DossierResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success")})
     List<DossierChangeEntry> changesSince(@RequestBody JSONPrimitive<OffsetDateTime> since);
 
+    @ResponseBody
+    @PostMapping(value = DOSSIER_REST_PATH + CHANGES_DETAILS_V2_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "See if there are changes to dossiers since param", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success")})
+    DossierChangeResponseV2 changesSinceV2(@RequestBody JSONPrimitive<OffsetDateTime> since);
+
 
     @ResponseBody
     @PostMapping(value = DOSSIER_REST_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@@ -95,6 +105,13 @@ public interface DossierResource {
     List<Dossier> getDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                               @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted);
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = DOSSIER_REST_PATH+BY_ID_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets dossiers by ids.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    JSONPrimitive<Map<String,Dossier>> getDossiersByIds(@RequestBody JSONPrimitive<Set<String>> dossierIds);
+
 
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DossierTemplateResource.java

@@ -95,8 +95,36 @@ public interface DossierTemplateResource {
 
     @ResponseStatus(value = HttpStatus.NO_CONTENT)
     @PostMapping(value = DOSSIER_TEMPLATE_PATH + IMPORT_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Receives an archive to import", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Archive have successfully imported"), @ApiResponse(responseCode = "400", description = "Validation failed during import"), @ApiResponse(responseCode = "404", description = "The dossier template to update does not exist")})
+    @Operation(summary = "Receives an archive to import", description = """
+            Import process stuck in one of the steps:"
+            0 - Reading the archive content
+            
+            1 - store information about the dossier template
+            
+            2 - store the colors
+            
+            3 - store the watermarks
+            
+            4 - store the dossier status
+            
+            5 - store the dossier attributes
+            
+            6 - store the file attributes
+            
+            7 - store the report templates
+            
+            8 - store the legal basis
+            
+            9 - store the file attribute configuration
+            
+            10 - store the component definitions
+            
+            11 - store the component mappings
+            
+            12 - store the types and entities
+            
+            13 - store the rules and component rules""")
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Archive have successfully imported"), @ApiResponse(responseCode = "400"), @ApiResponse(responseCode = "404", description = "The dossier template to update does not exist")})
     DossierTemplateModel importDossierTemplate(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                @RequestParam(value = DOSSIER_TEMPLATE_ID, required = false) String dossierTemplateId,
                                                @RequestParam(value = "updateExistingDossierTemplate", required = false, defaultValue = "false") boolean updateExistingDossierTemplate);

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/FileAttributesResource.java

@@ -44,7 +44,11 @@ public interface FileAttributesResource {
     String FILE_ID = "fileId";
     String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
 
-    Set<String> encodingList = Sets.newHashSet("ISO", "ASCII", "UTF-8");
+    String UTF_ENCODING = "UTF-8";
+    String ASCII_ENCODING = "ASCII";
+    String ISO_ENCODING = "ISO-8859-1";
+
+    Set<String> encodingList = Sets.newHashSet(ISO_ENCODING, ASCII_ENCODING, UTF_ENCODING);
 
 
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java

@@ -21,7 +21,7 @@ public interface ImageSimilaritySearchResource {
 
     @ResponseStatus(value = HttpStatus.OK)
     @Operation(summary = "Gets similiar images to given image", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found")})
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "400", description = "Bad request: missing parameter")})
     @PostMapping(value = IMAGE_SIMILARITY_SEARCH_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
     ResponseEntity<ImageSimilaritySearchResponse> getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest);
 

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/LegalBasisMappingResource.java

@@ -28,15 +28,15 @@ public interface LegalBasisMappingResource {
 
     @ResponseStatus(HttpStatus.NO_CONTENT)
     @PostMapping(value = LEGAL_BASIS_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE + DELETE_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "delete some legal basis by their names.", description = "None")
+    @Operation(summary = "delete some legal basis by their technical names.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK")})
-    void deleteLegalBasis(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @RequestBody List<String> legalBasisNames);
+    void deleteLegalBasis(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @RequestBody List<String> legalBasisTechnicalNames);
 
 
     @ResponseStatus(HttpStatus.NO_CONTENT)
     @PutMapping(value = LEGAL_BASIS_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Add or update one legalBasis.", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK")})
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "400", description = "Missing required parameter")})
     void addOrUpdateLegalBasis(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @RequestBody LegalBasis legalBasis);
 
 
@@ -50,7 +50,7 @@ public interface LegalBasisMappingResource {
     @ResponseBody
     @ResponseStatus(value = HttpStatus.OK)
     @GetMapping(value = LEGAL_BASIS_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Set the mapping between legal basis and redaction reason.", description = "None")
+    @Operation(summary = "Get all legal basis mapping in dossier template.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
     List<LegalBasis> getLegalBasisMapping(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId);
 

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ManualRedactionResource.java

@@ -16,11 +16,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.CommentResp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationComments;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionBulkLocalRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationBulkLocalRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionBulkLocalRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ResizeRedactionRequestModel;
 
@@ -95,17 +98,40 @@ public interface ManualRedactionResource {
                                              @RequestBody Set<AddRedactionRequestModel> addRedactionRequest);
 
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk-local/add"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Adds a bulk of local redactions", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    ManualRedactionResponse addRedactionBulkLocal(@PathVariable(DOSSIER_ID) String dossierId,
+                                                  @PathVariable(FILE_ID) String fileId,
+                                                  @RequestBody AddRedactionBulkLocalRequestModel addRedactionRequest);
+
+
+    @ResponseStatus(value = HttpStatus.OK)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk-local/remove"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Removes a bulk of local redactions", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    ManualRedactionResponse removeRedactionBulkLocal(@PathVariable(DOSSIER_ID) String dossierId,
+                                                     @PathVariable(FILE_ID) String fileId,
+                                                     @RequestBody RemoveRedactionBulkLocalRequestModel removeRedactionRequest);
+
+
     @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = MANUAL_REDACTION_REST_PATH
                          + "/bulk/redaction/remove"
                          + DOSSIER_ID_PATH_PARAM
                          + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Removes the redactions list", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse removeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                 @PathVariable(FILE_ID) String fileId,
-                                                @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests,
-                                                @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -139,11 +165,22 @@ public interface ManualRedactionResource {
                          + DOSSIER_ID_PATH_PARAM
                          + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Recategorizes the list of redaction log entries", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse recategorizeBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                              @PathVariable(FILE_ID) String fileId,
-                                             @RequestBody Set<RecategorizationRequestModel> recategorizationRequests,
-                                             @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                             @RequestBody Set<RecategorizationRequestModel> recategorizationRequests);
+
+
+    @ResponseStatus(value = HttpStatus.OK)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk-local/recategorize"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Recategorizes the list of redaction log entries", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    ManualRedactionResponse recategorizeBulkLocal(@PathVariable(DOSSIER_ID) String dossierId,
+                                                  @PathVariable(FILE_ID) String fileId,
+                                                  @RequestBody RecategorizationBulkLocalRequestModel recategorizationRequest);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -155,8 +192,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier  or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse resizeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                 @PathVariable(FILE_ID) String fileId,
-                                                @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests,
-                                                @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests);
 
 
     @ResponseStatus(value = HttpStatus.OK)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java

@@ -1,56 +0,0 @@
-package com.iqser.red.service.persistence.service.v1.api.external.resource;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
-
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
-
-public interface MigrationStatusResource {
-
-    String MIGRATION_STATUS_REST_PATH = ExternalApi.BASE_PATH + "/migration-status";
-    String START_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/start_migration";
-    String REVERT_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/revert_migration";
-    String RETRY_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/retry_migration";
-
-    String FILE_ID = "fileId";
-    String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
-
-    String DOSSIER_ID = "dossierId";
-    String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
-
-
-    @ResponseBody
-    @PostMapping(value = MIGRATION_STATUS_REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Show the status of the migration", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    MigrationStatusResponse migrationStatus();
-
-
-    @ResponseBody
-    @PostMapping(value = START_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
-    @Operation(summary = "Start SAAS migration for specific file", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> startMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
-
-
-    @ResponseBody
-    @PostMapping(value = REVERT_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
-    @Operation(summary = "Start SAAS migration for specific file", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> revertMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
-
-
-    @ResponseBody
-    @PostMapping(value = RETRY_MIGRATION_REST_PATH)
-    @Operation(summary = "Restart SAAS migration for all files in error state", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> requeueErrorFiles();
-
-}

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ReanalysisResource.java

@@ -38,6 +38,7 @@ public interface ReanalysisResource {
 
     String EXCLUDED_STATUS_PARAM = "excluded";
     String FORCE_PARAM = "force";
+    String ALL_PAGES = "allPages";
 
 
     @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE)
@@ -73,7 +74,8 @@ public interface ReanalysisResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "409", description = "Conflict"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden"), @ApiResponse(responseCode = "400", description = "Cannot OCR approved file")})
     void ocrFile(@PathVariable(DOSSIER_ID) String dossierId,
                  @PathVariable(FILE_ID) String fileId,
-                 @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force);
+                 @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force,
+                 @RequestParam(value = ALL_PAGES, required = false, defaultValue = FALSE) boolean allPages);
 
 
     @Operation(summary = "Ocr and reanalyze multiple files for a dossier", description = "None")

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RedactionLogResource.java

@@ -1,71 +0,0 @@
-package com.iqser.red.service.persistence.service.v1.api.external.resource;
-
-import java.util.List;
-
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseStatus;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.FilteredRedactionLogRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
-
-@Deprecated(forRemoval = true)
-@ResponseStatus(value = HttpStatus.OK)
-public interface RedactionLogResource {
-
-    String REDACTION_LOG_PATH = ExternalApi.BASE_PATH + "/redactionLog";
-    String SECTION_GRID_PATH = ExternalApi.BASE_PATH + "/sectionGrid";
-    String SECTION_TEXT_PATH = ExternalApi.BASE_PATH + "/sectionText";
-
-    String FILE_ID = "fileId";
-    String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
-
-    String DOSSIER_ID = "dossierId";
-    String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
-
-
-    @Deprecated(forRemoval = true)
-    @GetMapping(value = REDACTION_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Gets the redaction log for a fileId", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The redaction log is not found.")})
-    RedactionLog getRedactionLog(@PathVariable(DOSSIER_ID) String dossierId,
-                                 @PathVariable(FILE_ID) String fileId,
-                                 @RequestParam(value = "excludedType", required = false) List<String> excludedTypes,
-                                 @RequestParam(value = "withManualRedactions", required = false, defaultValue = "true") boolean withManualRedactions,
-                                 @RequestParam(value = "includeFalsePositives", required = false, defaultValue = "false") boolean includeFalsePositives);
-
-
-    @Deprecated(forRemoval = true)
-    @GetMapping(value = SECTION_GRID_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Gets the section grid for a fileId", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The section grid is not found.")})
-    SectionGrid getSectionGrid(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId);
-
-
-    @Deprecated(forRemoval = true)
-    @GetMapping(value = SECTION_TEXT_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE)
-    @Operation(summary = "Gets the text blocks of a document for a fileId", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The section text is not found.")})
-    ResponseEntity<?> getSectionText(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId);
-
-
-    @Deprecated(forRemoval = true)
-    @PostMapping(value = REDACTION_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + "/filtered", produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Gets the redaction log for a fileId grater than the specified date", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The redaction log is not found.")})
-    RedactionLog getFilteredRedactionLog(@PathVariable(DOSSIER_ID) String dossierId,
-                                         @PathVariable(FILE_ID) String fileId,
-                                         @RequestBody FilteredRedactionLogRequest filteredRedactionLogRequest);
-
-}

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RulesResource.java

@@ -6,6 +6,7 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RequestPart;
@@ -28,6 +29,7 @@ public interface RulesResource {
     String RULES_PATH = ExternalApi.BASE_PATH + "/rules";
     String UPLOAD_PATH = "/upload";
     String DOWNLOAD_PATH = "/download";
+    String RESET_PATH = "/reset";
 
     String DOSSIER_TEMPLATE_PARAMETER_NAME = "dossierTemplateId";
     String DOSSIER_TEMPLATE_PATH_VARIABLE = "/{dossierTemplateId}";
@@ -105,4 +107,10 @@ public interface RulesResource {
     @GetMapping(value = RULES_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE + RULE_FILE_TYPE_PATH_VARIABLE + DOWNLOAD_PATH)
     ResponseEntity<?> downloadFile(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType);
 
+    @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
+    @Operation(summary = "Resets the timeout detected flag in a Rule file.")
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "No content")})
+    @PutMapping(value = RULES_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE + RULE_FILE_TYPE_PATH_VARIABLE + RESET_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    void unlockRules(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType);
 }

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/StatusResource.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.external.resource;
 import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -16,6 +17,8 @@ import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.warning.ApproveResponse;
 
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
@@ -24,6 +27,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
 public interface StatusResource {
 
     String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status";
+    String BY_ID_PATH = "/by-id";
     String CHANGES_SINCE_PATH = "/changes";
     String BULK_REST_PATH = "/bulk";
     String ASSIGNEE_REST_PATH = "/set-assignee";
@@ -35,6 +39,7 @@ public interface StatusResource {
     String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
 
     String ASSIGNEE_ID_REQUEST_PARAM = "assigneeId";
+    String FORCE_REQUEST_PARAM = "force";
 
     String DELETED_PATH = "/softdeleted";
 
@@ -54,6 +59,14 @@ public interface StatusResource {
     Map<String, List<FileStatus>> getDossierStatus(@RequestBody List<String> dossierIds);
 
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = STATUS_REST_PATH + BY_ID_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets the status for files by dossierId and fileIds.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    JSONPrimitive<Map<String, List<FileStatus>>> getFilesByIds(@RequestBody JSONPrimitive<Map<String, Set<String>>> filesByDossier);
+
+
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody
     @PostMapping(value = STATUS_REST_PATH + DELETED_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
@@ -105,11 +118,13 @@ public interface StatusResource {
                                 @RequestParam(value = ASSIGNEE_ID_REQUEST_PARAM, required = false) String assigneeId);
 
 
-    @ResponseStatus(value = HttpStatus.NO_CONTENT)
+    @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = STATUS_REST_PATH + "/approved" + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE)
     @Operation(summary = "Sets the status APPROVED for a file.", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void setStatusApproved(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId);
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    ApproveResponse setStatusApproved(@PathVariable(DOSSIER_ID) String dossierId,
+                                      @PathVariable(FILE_ID) String fileId,
+                                      @RequestParam(value = FORCE_REQUEST_PARAM, required = false, defaultValue = "false") boolean force);
 
 
     @ResponseStatus(value = HttpStatus.NO_CONTENT)
@@ -139,11 +154,13 @@ public interface StatusResource {
                                        @RequestParam(value = ASSIGNEE_ID_REQUEST_PARAM, required = false) String assigneeId);
 
 
-    @ResponseStatus(value = HttpStatus.NO_CONTENT)
+    @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = STATUS_REST_PATH + "/approved" + DOSSIER_ID_PATH_VARIABLE + BULK_REST_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Sets the status APPROVED for a list of files.", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void setStatusApprovedForList(@PathVariable(DOSSIER_ID) String dossierId, @RequestBody List<String> fileIds);
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    List<ApproveResponse> setStatusApprovedForList(@PathVariable(DOSSIER_ID) String dossierId,
+                                             @RequestBody List<String> fileIds,
+                                             @RequestParam(value = FORCE_REQUEST_PARAM, required = false, defaultValue = "false") boolean force);
 
 
     @ResponseStatus(value = HttpStatus.NO_CONTENT)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalyzeFilesResponse;
 
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Schema;
@@ -33,6 +34,7 @@ public interface SupportResource {
 
     String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status/filter";
     String FILE_EXCHANGE_REST_PATH = ExternalApi.BASE_PATH + "/file-exchange";
+    String DATASET_EXCHANGE = ExternalApi.BASE_PATH + "/dataset-exchange";
 
     String BULK_REST_PATH = "/bulk";
 
@@ -49,10 +51,12 @@ public interface SupportResource {
     String FALSE = "false";
 
     String FULL_REANALYSIS_PARAM = "fullReanalysis";
+    String RUN_OCR_PARAM = "runOcr";
     String EXPORT = "/export";
     String IMPORT = "/import";
 
 
+    @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
     @Operation(summary = "Reanalyze all files in dossier template", description = """
             ## Reanalyze Files Endpoint
@@ -70,14 +74,15 @@ public interface SupportResource {
             - **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated.
             - **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null.
             """)
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings);
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    ReanalyzeFilesResponse reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings);
 
 
     @PostMapping(value = ERROR_REANALYSIS_REST_PATH)
     @Operation(summary = "Reanalyze all files in error state.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void reanalyzeAllErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
+    void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
+                                        @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr);
 
 
     @PostMapping(value = ERROR_REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE + BULK_REST_PATH)
@@ -85,7 +90,8 @@ public interface SupportResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     void reanalyzeErrorFilesBulkForDossier(@PathVariable(DOSSIER_ID) String dossierId,
                                            @RequestBody List<String> fileIds,
-                                           @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
+                                           @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis,
+                                           @RequestParam(value = RUN_OCR_PARAM, required = false, defaultValue = FALSE) boolean runOcr);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -141,6 +147,19 @@ public interface SupportResource {
     DownloadResponse exportFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody FileExchangeExportRequest exportRequest);
 
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = DATASET_EXCHANGE + EXPORT + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
+            ## Export Preview Files Endpoint
+            
+            Use this endpoint to export all files of a DossierTemplate as a Preview file containing all applied annotations as redaction annotations with additional data
+            The endpoint returns a String storageId, which is used to query the DownloadController for the export zip archive's status and to download the archive.
+            """)
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId);
+
+
     @ResponseBody
     @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@@ -148,4 +167,12 @@ public interface SupportResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
     ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
+
+    @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
+    @PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. Returns the resulting dossierTemplateId.")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    ImportResponse importDataset(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
+
 }

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/UserStatsResource.java

@@ -0,0 +1,37 @@
+package com.iqser.red.service.persistence.service.v1.api.external.resource;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.UserStats;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+
+@ApiResponses(value = {@ApiResponse(responseCode = "429", description = "Too many requests.")})
+public interface UserStatsResource {
+
+    String USER_ID_PARAM = "userId";
+
+    String USER_ID_PATH_VARIABLE = "/{" + USER_ID_PARAM + "}";
+
+    String STATS_PATH = "/user-stats";
+
+    String PATH = ExternalApi.BASE_PATH + STATS_PATH;
+
+
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @GetMapping(value = PATH + USER_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets user stats for user specified by id.", description = "")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found")})
+    ResponseEntity<UserStats> getUserStats(@Parameter(name = USER_ID_PARAM, description = "The unique identifier of the user whose statistics we want to retrieve.", required = true) @PathVariable(USER_ID_PARAM) String userId);
+
+}

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/BulkComponentsRequest.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.persistence.service.v2.api.external.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class BulkComponentsRequest {
+
+    private List<String> fileIds = new ArrayList<>();
+}

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/ComponentValue.java

@@ -20,9 +20,6 @@ public class ComponentValue {
     @NonNull
     String value;
     @JacksonXmlCData
-    @Deprecated
-    String originalValue;
-    @JacksonXmlCData
     String valueDescription;
     @JacksonXmlCData
     String componentRuleId;

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/DossierAttributeDefinition.java

@@ -17,5 +17,18 @@ public class DossierAttributeDefinition {
     private String name;
     private DossierAttributeType type;
     private String reportingPlaceholder;
+    private DossierDisplaySettings displaySettings;
+
+    @Data
+    @NoArgsConstructor
+    @AllArgsConstructor
+    @Builder
+    public static class DossierDisplaySettings {
+
+        private boolean editable;
+        private boolean filterable;
+        private boolean displayedInDossierList;
+
+    }
 
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/FileAttributeDefinition.java

@@ -20,6 +20,7 @@ public class FileAttributeDefinition {
     private String mappedCsvColumnHeader;
     private String reportingPlaceholder;
     private DisplaySettings displaySettings;
+    private boolean includeInCsvExport;
 
     @Data
     @NoArgsConstructor

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java

@@ -20,6 +20,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
+import com.iqser.red.service.persistence.service.v2.api.external.model.BulkComponentsRequest;
 import com.iqser.red.service.persistence.service.v2.api.external.model.Component;
 import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentOverrideList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents;
@@ -74,6 +75,16 @@ public interface ComponentResource {
                                               @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails);
 
 
+    @PostMapping(value = FILE_PATH
+                         + BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}, consumes = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Returns the components for all files of a dossier", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    FileComponentsList getComponentsForFiles(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
+                                             @Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier that contains the file.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                                             @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails,
+                                             @RequestBody BulkComponentsRequest bulkComponentsRequest);
+
+
     @ResponseBody
     @ResponseStatus(value = HttpStatus.NO_CONTENT)
     @PostMapping(value = FILE_PATH + FILE_ID_PATH_VARIABLE + OVERRIDES_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java

@@ -45,6 +45,7 @@ public interface DossierTemplateResource {
     String PATH = ExternalApiConstants.BASE_PATH + DOSSIER_TEMPLATE_PATH;
 
     String ENTITY_RULES_PATH = "/entity-rules";
+    String DATE_FORMATS_PATH = "/date-formats";
     String COMPONENT_RULES_PATH = "/component-rules";
     String COMPONENT_MAPPINGS_PATH = "/component-mappings";
     String DOSSIER_STATUS_DEFINITIONS_PATH = "/dossier-status-definitions";
@@ -61,6 +62,7 @@ public interface DossierTemplateResource {
     String DRY_RUN_PARAM = "dryRun";
     String ENCODING_PARAM = "encoding";
     String DELIMITER_PARAM = "delimiter";
+    String QUOTE_CHAR_PARAM = "quoteChar";
     String MAPPING_NAME_PARAM = "name";
 
     String INCLUDE_SOFT_DELETED = "includeSoftDeleted";
@@ -120,6 +122,22 @@ public interface DossierTemplateResource {
     ResponseEntity<?> downloadComponentRules(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId);
 
 
+    @ResponseStatus(value = HttpStatus.NO_CONTENT)
+    @PostMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + DATE_FORMATS_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Upload a date formats file for a specific DossierTemplate.")
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Date formats upload successful."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found."), @ApiResponse(responseCode = "400", description = "Uploaded date formats could not be verified."), @ApiResponse(responseCode = "422", description = "Uploaded date formats file could not be parsed.")})
+    ResponseEntity<?> uploadDateFormats(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
+                                        @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
+
+
+    @ResponseBody
+    @ResponseStatus(value = HttpStatus.NO_CONTENT)
+    @Operation(summary = "Returns file containing the currently used date formats.")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")})
+    @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + DATE_FORMATS_PATH)
+    ResponseEntity<?> downloadDateFormats(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId);
+
+
     @Operation(summary = "Get the file attribute definitions of a DossierTemplate.", description = "None")
     @GetMapping(value = PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE + FILE_ATTRIBUTE_DEFINITIONS_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "File attribute definitions returned successfully."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found.")})
@@ -141,7 +159,8 @@ public interface DossierTemplateResource {
                                                 @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                 @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
                                                 @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
-                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
+                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
+                                                @Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
 
 
     @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None")
@@ -155,7 +174,8 @@ public interface DossierTemplateResource {
                                                 @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                 @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
                                                 @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
-                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
+                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
+                                                @Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
 
 
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml

@@ -3,8 +3,12 @@ info:
   title: DocuMine API
   version: "1.0.0"
   description: |
+    # Introduction
+    
     The DocuMine API provides a comprehensive solution for managing resources such as dossiers and their associated files. 
     Users can also retrieve components of files that have been processed and extracted by the system. 
+    
+    # Authorization
 
     All endpoints are secured using OAuth2, with the "authorizationCode" being the general supported authorization flow.
     Obtain a JWT token for authentication and send it in the 'Authorization' header with the format `Bearer {JWT_TOKEN}`.
@@ -12,12 +16,13 @@ info:
     Please also note that the `authorizationUrl` and `tokenUrl` in this specification contain `{workspaceId}` placeholders that
     must be replaced by your respective DocuMine workspace identifier.
 
-    Example Headers:
+    **Example Headers:**
     ```properties
     Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI...
     ```
   license:
     name: knecon License
+    url: https://knecon.swiss/documine/license
   contact:
     name: knecon Service Desk
     email: support@documine.ai
@@ -273,8 +278,82 @@ paths:
           $ref: '#/components/responses/429'
         "500":
           $ref: '#/components/responses/500'
+  /api/dossier-templates/{dossierTemplateId}/date-formats:
+    get:
+      operationId: downloadDateFormats
+      tags:
+        - 1. Dossier Templates
+      summary: Download the date formats of a specific dossier template.
+      description: |
+        Utilize this endpoint to download the date formats of a designated dossier template. The file is named 'date-formats.txt'
+        and contains the set of date formats used within the dossier template.
+      parameters:
+        - $ref: '#/components/parameters/dossierTemplateId'
+      responses:
+        "200":
+          headers:
+            Content-Disposition:
+              schema:
+                type: string
+                example: attachment; filename*=utf-8''date-formats.txt
+          content:
+            text/plain; charset=utf-8:
+              schema:
+                type: string
+          description: |
+            Successfully downloaded the requested date formats file.
+        "400":
+          $ref: '#/components/responses/400'
+        "401":
+          $ref: '#/components/responses/401'
+        "403":
+          $ref: '#/components/responses/403'
+        "404":
+          $ref: '#/components/responses/404-dossier-template'
+        "429":
+          $ref: '#/components/responses/429'
+        "500":
+          $ref: '#/components/responses/500'
+    post:
+      operationId: uploadDateFormats
+      tags:
+        - 1. Dossier Templates
+      summary: Upload and validate a date formats file for a specific dossier template.
+      description: |
+        Utilize this endpoint to upload the date formats to a designated dossier template.
+        It will be validated first and if there are errors and the response code is not `200`, the file will not be saved.
+        In this case, the response object will contain the errors that occurred.
+      parameters:
+        - $ref: '#/components/parameters/dossierTemplateId'
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/UploadRequest'
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/DateFormatPatternErrorMessage'
+        "400":
+          $ref: '#/components/responses/400'
+        "401":
+          $ref: '#/components/responses/401'
+        "403":
+          $ref: '#/components/responses/403'
+        "404":
+          $ref: '#/components/responses/404-dossier-template'
+        "429":
+          $ref: '#/components/responses/429'
+        "500":
+          $ref: '#/components/responses/500'
   /api/dossier-templates/{dossierTemplateId}/dossier-status-definitions:
     get:
+      operationId: getDossierStatusDefinitions
       summary: Returns the list of all existing dossier status definitions
       tags:
         - 1. Dossier Templates
@@ -312,6 +391,7 @@ paths:
           $ref: '#/components/responses/500'
   /api/dossier-templates/{dossierTemplateId}/dossier-attribute-definitions:
     get:
+      operationId: getDossierAttributeDefinitions
       summary: Returns the list of all existing dossier attribute definitions
       tags:
         - 1. Dossier Templates
@@ -349,6 +429,7 @@ paths:
           $ref: '#/components/responses/500'
   /api/dossier-templates/{dossierTemplateId}/file-attribute-definitions:
     get:
+      operationId: getFileAttributeDefinitions
       summary: Returns the list of all existing file attribute definitions
       tags:
         - 1. Dossier Templates
@@ -428,7 +509,7 @@ paths:
         - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
         
         #### Customization Options
-        - Users can specify the delimiter and encoding used in the CSV file.
+        - Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
         
         #### Usage
         - The component mapping file can be utilized in component rules to relate components to existing master data.
@@ -452,6 +533,7 @@ paths:
         - $ref: '#/components/parameters/mappingName'
         - $ref: '#/components/parameters/encoding'
         - $ref: '#/components/parameters/delimiter'
+        - $ref: '#/components/parameters/quoteChar'
       responses:
         "200":
           content:
@@ -473,7 +555,7 @@ paths:
         "500":
           $ref: '#/components/responses/500'
 
-  /api/dossier-templates/{dossierTemplateId}/component-mappings/{comonentMappingId}:
+  /api/dossier-templates/{dossierTemplateId}/component-mappings/{componentMappingId}:
     get:
       operationId: downloadMappingFile
       tags:
@@ -528,7 +610,7 @@ paths:
         - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
         
         #### Customization Options
-        - Users can specify the delimiter and encoding used in the CSV file.
+        - Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
       tags:
         - 1. Dossier Templates
       requestBody:
@@ -542,6 +624,7 @@ paths:
         - $ref: '#/components/parameters/mappingName'
         - $ref: '#/components/parameters/encoding'
         - $ref: '#/components/parameters/delimiter'
+        - $ref: '#/components/parameters/quoteChar'
       responses:
         "200":
           content:
@@ -630,7 +713,7 @@ paths:
         "500":
           $ref: '#/components/responses/500'
     get:
-      operationId: getComponents
+      operationId: getComponentDefinitions
       tags:
         - Component Definitions
       summary: Get all component definitions
@@ -853,6 +936,7 @@ paths:
           $ref: '#/components/responses/500'
   /api/dossier-templates/{dossierTemplateId}/report-templates:
     get:
+      operationId: getReportTemplates
       summary: Returns the list of all available report templates
       tags:
         - 1. Dossier Templates
@@ -1195,6 +1279,7 @@ paths:
         - $ref: '#/components/parameters/dossierTemplateId'
         - $ref: '#/components/parameters/dossierId'
         - $ref: '#/components/parameters/keepManualChanges'
+        - $ref: '#/components/parameters/disableAutomaticAnalysis'
       requestBody:
         content:
           multipart/form-data:
@@ -1517,7 +1602,6 @@ paths:
       parameters:
         - $ref: '#/components/parameters/dossierTemplateId'
         - $ref: '#/components/parameters/dossierId'
-        - $ref: '#/components/parameters/fileId'
         - $ref: '#/components/parameters/includeComponentDetails'
       responses:
         "200":
@@ -1542,6 +1626,50 @@ paths:
           $ref: '#/components/responses/429'
         "500":
           $ref: '#/components/responses/500'
+    post:
+      operationId: getComponentsForFiles
+      tags:
+        - 4. Components
+      summary: Returns the FileComponents for requested files
+      description: |
+        This endpoint fetches components for the requested files by its ids. Like individual file components, 
+        these represent various aspects, metadata or content of the files. Entity and component rules define these components based on the file's
+        content. They can give a *structured view* on a document's text.
+        
+         To include detailed component information, set the `includeDetails` query parameter to `true`.
+      parameters:
+        - $ref: '#/components/parameters/dossierTemplateId'
+        - $ref: '#/components/parameters/dossierId'
+        - $ref: '#/components/parameters/includeComponentDetails'
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/BulkComponentsRequest'
+        required: true
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/FileComponentsList'
+            application/xml:
+              schema:
+                $ref: '#/components/schemas/FileComponentsList'
+          description: |
+            Successfully fetched components for all files in the dossier.
+        "400":
+          $ref: '#/components/responses/400'
+        "401":
+          $ref: '#/components/responses/401'
+        "403":
+          $ref: '#/components/responses/403'
+        "404":
+          $ref: '#/components/responses/404-dossier'
+        "429":
+          $ref: '#/components/responses/429'
+        "500":
+          $ref: '#/components/responses/500'
   /api/downloads:
     get:
       operationId: getDownloadStatusList
@@ -2032,6 +2160,17 @@ components:
         example: ','
         default: ','
       description: "The delimiter used as a separator in a csv file."
+    quoteChar:
+      name: quoteChar
+      required: false
+      in: query
+      schema:
+        type: string
+        minLength: 1
+        maxLength: 1
+        example: '"'
+        default: '"'
+      description: "The quoteChar used to quote fields in a csv file."
     mappingName:
       name: name
       required: false
@@ -2234,6 +2373,19 @@ components:
         overwriting a file:
         - `true`: The system keeps the manual changes on re-uploading (overwriting) the file.
         - `false` (default): Manual changes get discarded on re-uploading the file.
+    disableAutomaticAnalysis:
+      name: disableAutomaticAnalysis
+      in: query
+      required: false
+      schema:
+        default: false
+        type: boolean
+      style: form
+      explode: true
+      description: |
+        A toggle to disable automatic analysis, i.e., to just upload and process the file without applying any automatic annotations:
+        - `true`: The system does not apply any automatic text annotations on the file. Images are detected and annotated anyway.
+        - `false` (default): The system applies automatic annotations on the file.
     includeComponentDetails:
       name: includeDetails
       in: query
@@ -2665,6 +2817,16 @@ components:
                         entityRuleId: DEF.13.37
                         type: another_entity_type
                         page: 456
+    BulkComponentsRequest:
+      type: object
+      description: Request payload to get components for multiple files.
+      properties:
+        fileIds:
+          type: array
+          description: A list with unique identifiers of the files for which components should be retrieved.
+          items:
+            type: string
+            description: The unique identifier of a file.
     DossierStatusDefinition:
       type: object
       description: |
@@ -2756,6 +2918,8 @@ components:
             placeholder follows a specific format convention:
             `{{dossier.attribute.<name>}}` while the name is transformed into 'PascalCase' and does not contain 
             whitespaces. The placeholder is unique in a dossier template.
+        displaySettings:
+          $ref: '#/components/schemas/DossierAttributeDisplaySettings'
       required:
         - name
         - type
@@ -2764,6 +2928,35 @@ components:
         name: "Document Summary"
         type: "TEXT"
         reportingPlaceholder: "{{dossier.attribute.DocumentSummary}}"
+        displaySettings:
+          editable: true
+          filterable: false
+          displayedInDossierList: false
+    DossierAttributeDisplaySettings:
+      type: object
+      description: |
+        Display setting for the user interface. These settings control how the UI handles and presents the dossier attributes.
+      properties:
+        editable:
+          type: boolean
+          description: |
+            If `true`, the user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible.
+        filterable:
+          type: boolean
+          description: |
+            If `true`, the user interfaces add filter options to the dossier list.
+        displayedInDossierList:
+          type: boolean
+          description: |
+            if `true`, the user interfaces show the values in the dossier list.
+      required:
+        - editable
+        - filterable
+        - displayedInDossierList
+      example:
+        editable: true
+        filterable: true
+        displayedInDossierList: false
     FileAttributeDefinition:
       type: object
       description: |
@@ -2886,10 +3079,18 @@ components:
             name: "Dossier Summary"
             type: "TEXT"
             reportingPlaceholder: "{{dossier.attribute.DossierSummary}}"
+            displaySettings:
+              editable: true
+              filterable: false
+              displayedInFileList: false
           - id: "23e45678-e90b-12d3-a456-765114174321"
             name: "Comment"
             type: "TEXT"
             reportingPlaceholder: "{{dossier.attribute.Comment}}"
+            displaySettings:
+              editable: true
+              filterable: false
+              displayedInFileList: false
     FileAttributeDefinitionList:
       type: object
       description: A list of file attribute definitions.
@@ -3534,7 +3735,7 @@ components:
         modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6
         validFrom: 2020-01-01T00:00:00.000+00:00
         validTo: 2030-12-31T23:59:59.999+00:00
-        dossierTemplateStatus: ACTIVE
+        status: ACTIVE
         removeWatermark: false
         keepImageMetadata: true
         ocrByDefault: true
@@ -3655,7 +3856,7 @@ components:
             modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6
             validFrom: 2020-01-01T00:00:00.000+00:00
             validTo: 2030-12-31T23:59:59.999+00:00
-            dossierTemplateStatus: ACTIVE
+            status: ACTIVE
             removeWatermark: false
             keepImageMetadata: true
             ocrByDefault: true
@@ -3673,7 +3874,7 @@ components:
             modifiedBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d
             validFrom: 2023-01-01T00:00:00.000+00:00
             validTo: 2033-12-31T23:59:59.999+00:00
-            dossierTemplateStatus: ACTIVE
+            status: ACTIVE
             removeWatermark: true
             keepImageMetadata: true
             ocrByDefault: true
@@ -4233,6 +4434,23 @@ components:
             type: string
           type: array
       type: object
+    DateFormatPatternErrorMessage:
+      description: Object containing information about a parsed data format pattern.
+      example:
+        line: 123
+        pattern: "dd.mm.yyyz"
+        message: "No valid pattern, character 'z' not recognized"
+      properties:
+        line:
+          description: The line number where the error or warning occurs.
+          format: int32
+          type: integer
+        pattern:
+          description: The pattern that could not be parsed.
+          type: string
+        message:
+          description: The error or warning message that describes the details.
+          type: string
     RuleValidationMessage:
       description: Object containing information about an uploaded rules file.
       example:

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml

@@ -3,10 +3,14 @@ info:
   title: RedactManager API
   version: "4.1.0-draft"
   description: |
+    # Introduction
+    
     The RedactManager API provides a comprehensive solution for managing resources such as dossiers and their associated files. 
     Users can also retrieve the results package for files that have been processed by the system and reviewed by the users.
     The results package can contain the optimized PDF file, the preview PDF, the redacted PDF and correlating redaction reports 
     in different formats.
+    
+    # Authorization
 
     All endpoints are secured using OAuth2, with the "authorizationCode" being the general supported authorization flow.
     Obtain a JWT token for authentication and send it in the 'Authorization' header with the format `Bearer {JWT_TOKEN}`.
@@ -14,12 +18,13 @@ info:
     Please also note that the `authorizationUrl` and `tokenUrl` in this specification contain `{workspaceId}` placeholders that
     must be replaced by your respective RedactManager workspace identifier.
 
-    Example Headers:
+    **Example Headers:**
     ```properties
     Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI...
     ```
   license:
     name: knecon License
+    url: https://knecon.swiss/redactmanager/license
   contact:
     name: knecon Service Desk
     email: support@redactmanager.com
@@ -115,6 +120,7 @@ paths:
           $ref: '#/components/responses/500'
   /api/dossier-templates/{dossierTemplateId}/dossier-status-definitions:
     get:
+      operationId: getDossierStatusDefinitions
       summary: Returns the list of all existing dossier status definitions
       tags:
         - 1. Dossier Templates
@@ -152,6 +158,7 @@ paths:
           $ref: '#/components/responses/500'
   /api/dossier-templates/{dossierTemplateId}/dossier-attribute-definitions:
     get:
+      operationId: getDossierAttributeDefinitions
       summary: Returns the list of all existing dossier attribute definitions
       tags:
         - 1. Dossier Templates
@@ -189,11 +196,12 @@ paths:
           $ref: '#/components/responses/500'
   /api/dossier-templates/{dossierTemplateId}/file-attribute-definitions:
     get:
+      operationId: getFileAttributeDefinitions
       summary: Returns the list of all existing file attribute definitions
       tags:
         - 1. Dossier Templates
       description: |
-        Retrieves a collection of file attribute definitions associated with a specific dossier template. Each file
+        Use this endpoint to retrieve a collection of file attribute definitions associated with a specific dossier template. Each file
         attribute definition includes details such as attribute type, name, and other relevant metadata. This endpoint 
         is useful for clients needing to understand what attributes are expected or allowed for files associated with 
         a specific dossier template.
@@ -221,6 +229,7 @@ paths:
           $ref: '#/components/responses/500'
   /api/dossier-templates/{dossierTemplateId}/report-templates:
     get:
+      operationId: getReportTemplates
       summary: Returns the list of all available report templates
       tags:
         - 1. Dossier Templates
@@ -415,7 +424,7 @@ paths:
         - 2. Dossiers
       summary: Update or set attributes for a specific dossier.
       description: |
-        This endpoint facilitates the updating or setting of specific dossier attributes for a given dossier. 
+        Use this endpoint to update or set dossier attributes for a given dossier. 
         Ensure you provide the necessary dossier attributes within the request body.
 
         Use this route to maintain or enhance dossier metadata and properties.
@@ -565,6 +574,7 @@ paths:
         - $ref: '#/components/parameters/dossierTemplateId'
         - $ref: '#/components/parameters/dossierId'
         - $ref: '#/components/parameters/keepManualChanges'
+        - $ref: '#/components/parameters/disableAutomaticAnalysis'
       requestBody:
         content:
           multipart/form-data:
@@ -1062,14 +1072,14 @@ components:
   responses:
     "400":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
         The request was malformed or invalid. Double-check the request structure or parameters.
     "401":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
@@ -1077,21 +1087,21 @@ components:
         resource. This error can happen if the access token was revoked or has expired.
     "403":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
         Forbidden. Your credentials are valid, but you do not have permission to access this resource.
     "404-dossier-template":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
         Dossier template not found. This happens if the requested dossier template does not exist.
     "404-dossier":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
@@ -1101,7 +1111,7 @@ components:
         for a previously existing dossier only if it is actually deleted permanently.
     "404-file":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
@@ -1111,35 +1121,35 @@ components:
         only if the file is deleted permanently.
     "404-download":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
         Download not found. This happens if the requested download does not exist for the current user.
     "404-user":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
         User not found. This happens if the requested user does not exist.
     "409-dossier-conflict":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
         Name conflict: The provided name is already in use by another dossier. It needs to be unique in the scope of your workspace.
     "429":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: |
         Too many requests have been made in a given time frame. Rate limiting is in place to prevent abuse.
     "500":
       content:
-        '*/*':
+        'application/json':
           schema:
             $ref: '#/components/schemas/ErrorMessage'
       description: Internal Server Error. An unexpected error occurred on the server side. Please try again later or contact support.
@@ -1153,18 +1163,6 @@ components:
       style: simple
       explode: false
       description: The identifier of a dossier template
-    dryRun:
-      name: dryRun
-      in: query
-      required: false
-      schema:
-        default: false
-        type: boolean
-      style: form
-      explode: true
-      description: |
-        A toggle to activate the dry-run mode: If set to `false` (default), the request will update the system. 
-        If set to `true`, the request will just be evaluated without actual changes in the system.
     dossierId:
       name: dossierId
       in: path
@@ -1359,6 +1357,19 @@ components:
         overwriting a file:
         - `true`: The system keeps the manual changes on re-uploading (overwriting) the file.
         - `false` (default): Manual changes get discarded on re-uploading the file.
+    disableAutomaticAnalysis:
+      name: disableAutomaticAnalysis
+      in: query
+      required: false
+      schema:
+        default: false
+        type: boolean
+      style: form
+      explode: true
+      description: |
+        A toggle to disable automatic analysis, i.e., to just upload and process the file without applying any automatic redactions:
+        - `true`: The system does not apply any automatic redactions on the file. However, if the file contains redaction annotations, they are still imported.
+        - `false` (default): The system applies automatic redactions on the file.
     downloadId:
       name: downloadId
       in: path
@@ -1390,37 +1401,6 @@ components:
         If the `username` parameter is set, the user list is filtered for that specific user name. This means the list
         either has one matching entry or is empty.
   schemas:
-    EntityReference:
-      type: object
-      description: |
-        Represents a reference to an entity object discovered in a document. Together with the unique identifier
-        of the entity, the reference contains some basic details of the entity.
-      properties:
-        id:
-          type: string
-          description: |
-            The unique identifier for the entity.
-            
-            > **Note:** In general, it is a valid UUID. Only if an entity spans over multiple pages, it is split on page breaks. Each
-            > part becomes an own entity object and the first one keeps the UUID while the following get an additional suffix.
-            > 
-            > Example: Even if `bcd22239-c3df-442f-a5a1-1664cba94dc6_2` is not a valid UUID it is still a valid identifier for an
-            > entity object.
-        entityRuleId:
-          type: string
-          description: An identifier that represents a specific rule associated with the entity.
-        type:
-          type: string
-          description: The name of the entity.
-        page:
-          type: integer
-          format: int32
-          description: The page number in the document where the entity is located.
-      example:
-        id: bcd22239-cedf-442f-a5a1-1664cba94dc6
-        entityRuleId: ABC.0.0
-        type: entity_type
-        page: 123
     DossierStatusDefinition:
       type: object
       description: |
@@ -1512,6 +1492,8 @@ components:
             placeholder follows a specific format convention:
             `{{dossier.attribute.<name>}}` while the name is transformed into 'PascalCase' and does not contain 
             whitespaces. The placeholder is unique in a dossier template.
+        displaySettings:
+          $ref: '#/components/schemas/DossierAttributeDisplaySettings'
       required:
         - name
         - type
@@ -1520,6 +1502,35 @@ components:
         name: "Document Summary"
         type: "TEXT"
         reportingPlaceholder: "{{dossier.attribute.DocumentSummary}}"
+        displaySettings:
+          editable: true
+          filterable: false
+          displayedInDossierList: false
+    DossierAttributeDisplaySettings:
+      type: object
+      description: |
+        Display setting for the user interface. These settings control how the UI handles and presents the dossier attributes.
+      properties:
+        editable:
+          type: boolean
+          description: |
+            If `true`, the user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible.
+        filterable:
+          type: boolean
+          description: |
+            If `true`, the user interfaces add filter options to the dossier list.
+        displayedInDossierList:
+          type: boolean
+          description: |
+            if `true`, the user interfaces show the values in the dossier list.
+      required:
+        - editable
+        - filterable
+        - displayedInDossierList
+      example:
+        editable: true
+        filterable: true
+        displayedInDossierList: false
     FileAttributeDefinition:
       type: object
       description: |
@@ -1642,10 +1653,18 @@ components:
             name: "Dossier Summary"
             type: "TEXT"
             reportingPlaceholder: "{{dossier.attribute.DossierSummary}}"
+            displaySettings:
+              editable: true
+              filterable: false
+              displayedInFileList: false
           - id: "23e45678-e90b-12d3-a456-765114174321"
             name: "Comment"
             type: "TEXT"
             reportingPlaceholder: "{{dossier.attribute.Comment}}"
+            displaySettings:
+              editable: true
+              filterable: false
+              displayedInFileList: false
     FileAttributeDefinitionList:
       type: object
       description: A list of file attribute definitions.
@@ -2096,7 +2115,7 @@ components:
         modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6
         validFrom: 2020-01-01T00:00:00.000+00:00
         validTo: 2030-12-31T23:59:59.999+00:00
-        dossierTemplateStatus: ACTIVE
+        status: ACTIVE
         removeWatermark: false
         keepImageMetadata: false
         ocrByDefault: false
@@ -2191,7 +2210,7 @@ components:
         Specifies the types of files that will part of the created download package. The defaults can be defined in the dossier template
         and can be overwritten individually on each download.
         
-        RedactManager supports `ORIGINAL`, `PREVIEW`, `DELTA_PREVIEW`, and `REDACTED`:
+        RedactManager supports `ORIGINAL`, `PREVIEW`, `OPTIMIZED_PREVIEW`, `DELTA_PREVIEW`, and `REDACTED`:
         
         - `ORIGINAL` Contrary to intuition, this is not the uploaded file, but the pre-processed,
           optimized PDF, which may also contain the OCR results.
@@ -2200,6 +2219,8 @@ components:
           system. Note that the content to redact is actually still present and readable. 
           The redaction information is embedded so you can restore the redactions when uploading the
           `PREVIEW` PDF into RedactManager.
+        - `OPTIMIZED_PREVIEW` The optimization process removes bookmarks and links in the document.
+          Other than that, this is the same as the Preview PDF. 
         - `DELTA_PREVIEW` Shows changes if redactions were imported (e.g., by uploading a Preview PDF
           with redaction annotations):
           Unchanged imported redactions are highlighted in green, removed imported redactions are 
@@ -2210,6 +2231,7 @@ components:
         enum:
           - ORIGINAL
           - PREVIEW
+          - OPTIMIZED_PREVIEW
           - DELTA_PREVIEW
           - REDACTED
         type: string
@@ -2229,7 +2251,7 @@ components:
             modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6
             validFrom: 2020-01-01T00:00:00.000+00:00
             validTo: 2030-12-31T23:59:59.999+00:00
-            dossierTemplateStatus: ACTIVE
+            status: ACTIVE
             removeWatermark: false
             keepImageMetadata: false
             ocrByDefault: false
@@ -2248,7 +2270,7 @@ components:
             modifiedBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d
             validFrom: 2023-01-01T00:00:00.000+00:00
             validTo: 2033-12-31T23:59:59.999+00:00
-            dossierTemplateStatus: ACTIVE
+            status: ACTIVE
             removeWatermark: true
             keepImageMetadata: false
             ocrByDefault: true

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DateFormatsInternalController.java

@@ -0,0 +1,46 @@
+package com.iqser.red.service.persistence.v1.internal.api.controller;
+
+import org.springframework.web.bind.annotation.RestController;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DateFormatsEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.RulesTimeoutDetectedException;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.DateFormatsResource;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+
+import lombok.RequiredArgsConstructor;
+
+@RestController
+@RequiredArgsConstructor
+public class DateFormatsInternalController implements DateFormatsResource {
+
+    private final DateFormatsPersistenceService dateFormatsPersistenceService;
+
+
+    @Override
+    public JSONPrimitive<String> getDateFormats(String dossierTemplateId) {
+
+        DateFormatsEntity dateFormats = getDateFormatsForDossierTemplate(dossierTemplateId);
+        return new JSONPrimitive<>(dateFormats.getValue());
+    }
+
+
+    @Override
+    public long getVersion(String dossierTemplateId) {
+
+        DateFormatsEntity dateFormats = getDateFormatsForDossierTemplate(dossierTemplateId);
+        return dateFormats.getVersion();
+    }
+
+
+    private DateFormatsEntity getDateFormatsForDossierTemplate(String dossierTemplateId) {
+
+        var dateFormatsOptional = dateFormatsPersistenceService.getDateFormats(dossierTemplateId);
+        if (dateFormatsOptional.isEmpty()) {
+            throw new NotFoundException(String.format("No date formats file found for dossierTemplateId %s", dossierTemplateId));
+        }
+        return dateFormatsOptional.get();
+    }
+
+}

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierAttributesConfigInternalController.java

@@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.DossierAttributeConfigMapper;
 import com.iqser.red.service.persistence.service.v1.api.internal.resources.DossierAttributesConfigResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
 
@@ -27,7 +28,7 @@ public class DossierAttributesConfigInternalController implements DossierAttribu
     @Override
     public List<DossierAttributeConfig> getDossierAttributes(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
 
-        return convert(dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId), DossierAttributeConfig.class);
+        return convert(dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId), DossierAttributeConfig.class, new DossierAttributeConfigMapper());
     }
 
 }

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileAttributesConfigInternalController.java

@@ -8,6 +8,7 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileAttributeConfigMapper;
 import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileAttributesConfigResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
 
@@ -25,7 +26,7 @@ public class FileAttributesConfigInternalController implements FileAttributesCon
     @Override
     public List<FileAttributeConfig> getFileAttributeConfigs(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
 
-        return convert(fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId), FileAttributeConfig.class);
+        return convert(fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId), FileAttributeConfig.class, new FileAttributeConfigMapper());
     }
 
 }

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java

@@ -5,7 +5,6 @@ import java.time.temporal.ChronoUnit;
 
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
@@ -39,7 +38,7 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP
         fileStatusProcessingUpdateService.preprocessingFailed(dossierId,
                                                               fileId,
                                                               new FileErrorInfo("preprocessing failed",
-                                                                                MessagingConfiguration.PRE_PROCESSING_DLQ,
+                                                                                MessagingConfiguration.PREPROCESSING_DLQ,
                                                                                 "pdftron-service",
                                                                                 OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
     }
@@ -125,7 +124,7 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP
         fileStatusProcessingUpdateService.indexingFailed(dossierId,
                                                          fileId,
                                                          new FileErrorInfo("indexing failed",
-                                                                           MessagingConfiguration.INDEXING_DQL,
+                                                                           MessagingConfiguration.INDEXING_DLQ,
                                                                            "search-service",
                                                                            OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
     }

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/InternalComponentController.java

@@ -19,9 +19,9 @@ public class InternalComponentController implements InternalComponentResource {
 
 
     @Override
-    public ComponentLog getComponentLog(String dossierId, String fileId, boolean includeOverrides) {
+    public ComponentLog getComponentLog(String dossierId, String fileId) {
 
-        return componentLogService.getComponentLog(dossierId, fileId, includeOverrides);
+        return componentLogService.getComponentLog(dossierId, fileId);
 
     }
 

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/LicenseInternalController.java

@@ -0,0 +1,40 @@
+package com.iqser.red.service.persistence.v1.internal.api.controller;
+
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_LICENSE;
+import static com.iqser.red.service.persistence.management.v1.processor.service.LicenseUtilityService.LICENSE_OBJECT_ID;
+
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.iqser.red.service.persistence.management.v1.processor.service.LicenseUtilityService;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.LicenseResource;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.license.RedactionLicenseModel;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.RequiredArgsConstructor;
+
+@RestController
+@RequiredArgsConstructor
+public class LicenseInternalController implements LicenseResource {
+
+    private final StorageService storageService;
+    private final LicenseUtilityService licenseUtilityService;
+
+
+    @Override
+    public RedactionLicenseModel getLicense() {
+
+        String tenantId = TenantContext.getTenantId();
+        if (storageService.objectExists(tenantId, LICENSE_OBJECT_ID)) {
+            try {
+                return storageService.readJSONObject(tenantId, LICENSE_OBJECT_ID, RedactionLicenseModel.class);
+            } catch (Exception e) {
+                return licenseUtilityService.generateDemoLicense();
+            }
+        } else {
+            return licenseUtilityService.generateDemoLicense();
+        }
+    }
+
+}

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/RedactionLogInternalController.java

@@ -1,31 +0,0 @@
-package com.iqser.red.service.persistence.v1.internal.api.controller;
-
-import java.util.List;
-
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService;
-import com.iqser.red.service.persistence.service.v1.api.internal.resources.RedactionLogResource;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-
-import lombok.RequiredArgsConstructor;
-
-@RestController
-@RequiredArgsConstructor
-public class RedactionLogInternalController implements RedactionLogResource {
-
-    private final RedactionLogService redactionLogService;
-
-
-    public RedactionLog getRedactionLog(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
-                                        @PathVariable(FILE_ID) String fileId,
-                                        @RequestParam(value = "excludedType", required = false) List<String> excludedTypes,
-                                        @RequestParam(value = "withManualRedactions", required = false, defaultValue = "true") boolean withManualRedactions,
-                                        @RequestParam(value = "includeFalsePositives", required = false, defaultValue = "false") boolean includeFalsePositives) {
-
-        return redactionLogService.getRedactionLog(dossierId, fileId, excludedTypes);
-    }
-
-}

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/RulesInternalController.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.v1.internal.api.controller;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.RulesTimeoutDetectedException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.internal.resources.RulesResource;
@@ -22,20 +23,27 @@ public class RulesInternalController implements RulesResource {
     public JSONPrimitive<String> getRules(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
                                           @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType) {
 
-        return new JSONPrimitive<>(rulesPersistenceService.getRules(dossierTemplateId, ruleFileType).getValue());
+        var rulesOptional = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
+        if (rulesOptional.isEmpty()) {
+            throw new NotFoundException(String.format("No rule file of type %s found for dossierTemplateId %s", ruleFileType, dossierTemplateId));
+        }
+        return new JSONPrimitive<>(rulesOptional.get().getValue());
     }
 
 
     @Override
     public long getVersion(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType) {
 
-        var rules = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
+        var rulesOptional = rulesPersistenceService.getRules(dossierTemplateId, ruleFileType);
+        if (rulesOptional.isEmpty()) {
+            throw new NotFoundException(String.format("No rule file of type %s found for dossierTemplateId %s", ruleFileType, dossierTemplateId));
+        }
 
-        if (rules.isTimeoutDetected()) {
+        if (rulesOptional.get().isTimeoutDetected()) {
             throw new RulesTimeoutDetectedException(dossierTemplateId);
         }
 
-        return rules.getVersion();
+        return rulesOptional.get().getVersion();
     }
 
 

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java

@@ -35,17 +35,9 @@ public class AdminInterfaceController {
     @PostMapping("/reset-file")
     public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
 
-        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
-        fileManagementStorageService.deleteEntityLog(dossierId, fileId);
-        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
-        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
-        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
-        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
-        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
-        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.FIGURE);
-        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TABLES);
+        fileManagementStorageService.deleteAllObjects(dossierId, fileId);
 
-        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
+        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
 
     }
 
@@ -65,7 +57,7 @@ public class AdminInterfaceController {
 
         fileStatusService.validateFileIsNotDeletedAndNotApproved(fileId);
 
-        fileStatusService.setStatusOcrQueued(dossierId, fileId);
+        fileStatusService.setStatusOcrQueued(dossierId, fileId, false);
 
     }
 
@@ -99,7 +91,7 @@ public class AdminInterfaceController {
 
                     if (!dryRun) {
                         fileStatusService.validateFileIsNotDeletedAndNotApproved(file.getId());
-                        fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId());
+                        fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId(), false);
                     }
 
                 }
@@ -142,13 +134,9 @@ public class AdminInterfaceController {
 
                         fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
                         fileManagementStorageService.deleteEntityLog(dossierId, fileId);
-                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
-                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
-                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
-                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
-                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
+                        fileManagementStorageService.deleteDocumentAndNerObjects(dossierId, fileId);
 
-                        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
+                        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true, false);
                     }
 
                 }

persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DateFormatsResource.java

@@ -0,0 +1,32 @@
+package com.iqser.red.service.persistence.service.v1.api.internal.resources;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+
+public interface DateFormatsResource {
+
+    String PATH = "/date-formats";
+
+    String DOSSIER_TEMPLATE_PARAMETER_NAME = "dossierTemplateId";
+    String DOSSIER_TEMPLATE_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_PARAMETER_NAME + "}";
+
+    String VERSION_PATH = "/version";
+
+
+    @ResponseBody
+    @ResponseStatus(HttpStatus.OK)
+    @GetMapping(value = InternalApi.BASE_PATH + PATH + DOSSIER_TEMPLATE_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
+    JSONPrimitive<String> getDateFormats(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId);
+
+
+    @ResponseStatus(HttpStatus.OK)
+    @GetMapping(value = InternalApi.BASE_PATH + PATH + DOSSIER_TEMPLATE_PATH_VARIABLE + VERSION_PATH)
+    long getVersion(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId);
+
+}

persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/InternalComponentResource.java

@@ -4,7 +4,6 @@ import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
@@ -29,8 +28,6 @@ public interface InternalComponentResource {
     @GetMapping(value = COMPONENT_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Gets the component log for a fileId", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The component log is not found.")})
-    ComponentLog getComponentLog(@PathVariable(DOSSIER_ID) String dossierId,
-                                 @PathVariable(FILE_ID) String fileId,
-                                 @RequestParam(name = "includeOverrides", defaultValue = "true") boolean includeOverrides);
+    ComponentLog getComponentLog(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId);
 
 }

persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/LicenseResource.java

@@ -0,0 +1,25 @@
+package com.iqser.red.service.persistence.service.v1.api.internal.resources;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.license.RedactionLicenseModel;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+
+public interface LicenseResource {
+
+    String LICENSE_REST_PATH = InternalApi.BASE_PATH + "/license";
+
+    @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
+    @GetMapping(value = LICENSE_REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Returns the current license (or a demo license in case none is configured")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "License returned successfully.")})
+    RedactionLicenseModel getLicense();
+}

persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/RedactionLogResource.java

@@ -1,33 +0,0 @@
-package com.iqser.red.service.persistence.service.v1.api.internal.resources;
-
-import java.util.List;
-
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseStatus;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-
-@ResponseStatus(value = HttpStatus.OK)
-public interface RedactionLogResource {
-
-    String REDACTION_LOG_PATH = "/redactionLog";
-
-    String FILE_ID = "fileId";
-    String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
-
-    String DOSSIER_ID_PARAM = "dossierId";
-    String DOSSIER_ID_PATH_PARAM = "/{" + DOSSIER_ID_PARAM + "}";
-
-
-    @GetMapping(value = InternalApi.BASE_PATH + REDACTION_LOG_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-    RedactionLog getRedactionLog(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
-                                 @PathVariable(FILE_ID) String fileId,
-                                 @RequestParam(value = "excludedType", required = false) List<String> excludedTypes,
-                                 @RequestParam(value = "withManualRedactions", required = false, defaultValue = "true") boolean withManualRedactions,
-                                 @RequestParam(value = "includeFalsePositives", required = false, defaultValue = "false") boolean includeFalsePositives);
-
-}

persistence-service-v1/persistence-service-processor-v1/build.gradle.kts

@@ -14,6 +14,7 @@ configurations {
     }
 }
 dependencies {
+
     api(project(":persistence-service-shared-api-v1"))
     api(project(":persistence-service-shared-mongo-v1"))
     api(project(":persistence-service-external-api-v1"))
@@ -30,21 +31,23 @@ dependencies {
         exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
         exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
     }
-    api("com.knecon.fforesight:layoutparser-service-internal-api:0.149.0") {
-        exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
-        exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
-    }
     api("com.iqser.red.service:search-service-api-v1:${rootProject.extra.get("searchServiceVersion")}") {
         exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
         exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
     }
-    api("com.knecon.fforesight.service:ocr-service-api:4.24.0") {
-        exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
-        exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
+    api("com.knecon.fforesight:azure-ocr-service-api:0.13.0")
+    implementation("com.knecon.fforesight:llm-service-api:1.20.0-RED10072.2")
+    api("com.knecon.fforesight:jobs-commons:0.13.0")
+    api("com.iqser.red.commons:storage-commons:2.50.0")
+    api("com.knecon.fforesight:tenant-commons:0.31.0-RED10196.0") {
+        exclude(group = "com.iqser.red.commons", module = "storage-commons")
+    }
+    api("com.knecon.fforesight:database-tenant-commons:0.31.0") {
+        exclude(group = "com.knecon.fforesight", module = "tenant-commons")
+    }
+    api("com.knecon.fforesight:keycloak-commons:0.30.0") {
+        exclude(group = "com.knecon.fforesight", module = "tenant-commons")
     }
-    api("com.knecon.fforesight:jobs-commons:0.10.0")
-    api("com.knecon.fforesight:database-tenant-commons:0.24.0")
-    api("com.knecon.fforesight:keycloak-commons:0.29.0")
     api("com.knecon.fforesight:tracing-commons:0.5.0")
     api("com.knecon.fforesight:swagger-commons:0.7.0")
     api("com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:0.4.0")
@@ -58,7 +61,6 @@ dependencies {
     api("org.springframework.security:spring-security-messaging:6.1.3")
     api("com.iqser.red.commons:spring-commons:2.1.0")
     api("com.iqser.red.commons:jackson-commons:2.3.0")
-    api("com.iqser.red.commons:storage-commons:2.49.0")
     api("com.iqser.red.commons:spring-boot-starter-web-custom-commons:2.1.0")
     api("com.iqser.red.commons:metric-commons:2.3.0")
     api("org.apache.commons:commons-compress:1.21")
@@ -69,8 +71,8 @@ dependencies {
     api("commons-validator:commons-validator:1.7")
     api("com.opencsv:opencsv:5.9")
 
-    implementation("org.mapstruct:mapstruct:1.5.5.Final")
-    annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final")
+    implementation("org.mapstruct:mapstruct:1.6.2")
+    annotationProcessor("org.mapstruct:mapstruct-processor:1.6.2")
 
     testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
     testImplementation("org.testcontainers:postgresql:1.17.1")

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/AbstractACLService.java

@@ -3,12 +3,15 @@ package com.iqser.red.service.persistence.management.v1.processor.acl;
 import java.io.Serializable;
 
 import org.springframework.security.acls.domain.ObjectIdentityImpl;
+import org.springframework.security.acls.model.AlreadyExistsException;
 import org.springframework.security.acls.model.MutableAcl;
 import org.springframework.security.acls.model.MutableAclService;
 import org.springframework.security.acls.model.NotFoundException;
 
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @RequiredArgsConstructor
 public abstract class AbstractACLService<ID extends Serializable> {
 
@@ -21,7 +24,16 @@ public abstract class AbstractACLService<ID extends Serializable> {
         try {
             return (MutableAcl) mutableAclService.readAclById(objectIdentity);
         } catch (NotFoundException e) {
-            return mutableAclService.createAcl(objectIdentity);
+            try {
+                return mutableAclService.createAcl(objectIdentity);
+            } catch (AlreadyExistsException e1) {
+                // This happened always for the same dossierId repeatedly in an endless loop triggered by SyncUserPermissionsJob every 2 mins on prod stack till it broke the stack,
+                // so I don't think it was any kind of race condition.
+                // Maybe this happens due corrupt entries.
+                log.warn("Recreate already existing acl object {}", objectIdentity);
+                mutableAclService.deleteAcl(objectIdentity, true);
+                return mutableAclService.createAcl(objectIdentity);
+            }
         }
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/RedObjectIdentityRetrievalStrategy.java

@@ -5,6 +5,7 @@ import org.springframework.security.acls.model.ObjectIdentity;
 import org.springframework.security.acls.model.ObjectIdentityRetrievalStrategy;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.IHavingDossierId;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 
 import lombok.extern.slf4j.Slf4j;
@@ -18,8 +19,8 @@ public class RedObjectIdentityRetrievalStrategy implements ObjectIdentityRetriev
         log.debug("Requesting data for object: {}", domainObject);
         if (domainObject instanceof Dossier) {
             return new ObjectIdentityImpl("Dossier", ((Dossier) domainObject).getId());
-        } else if (domainObject instanceof DossierChangeEntry) {
-            return new ObjectIdentityImpl("Dossier", ((DossierChangeEntry) domainObject).getDossierId());
+        } else if (domainObject instanceof IHavingDossierId) {
+            return new ObjectIdentityImpl("Dossier", ((IHavingDossierId) domainObject).getDossierId());
         } else if (domainObject instanceof String) {
             // TODO ACL this will not work once we have more than one type.
             return new ObjectIdentityImpl("Dossier", (String) domainObject);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/api/ICustomPermissionService.java

@@ -33,7 +33,6 @@ public abstract class ICustomPermissionService<T, ID extends Serializable> exten
     }
 
 
-    @Transactional
     public void applyCustomPermissions(CustomPermissionMappingModel customPermissionMappingModel) {
 
         var objectIds = provideAllObjectIds();
@@ -44,7 +43,6 @@ public abstract class ICustomPermissionService<T, ID extends Serializable> exten
     protected abstract Collection<ID> provideAllObjectIds();
 
 
-    @Transactional
     public void applyCustomPermissions(CustomPermissionMappingModel customPermissionMappingModel, Serializable objectId) {
 
         var acl = getOrCreateACL(objectId);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/dossier/DossierACLService.java

@@ -73,6 +73,24 @@ public class DossierACLService extends AbstractACLService<String> {
     }
 
 
+    public Set<String> getOwners(String dossierId) {
+
+        ObjectIdentityImpl dossierIdentity = new ObjectIdentityImpl(getIdentifier(), dossierId);
+        var acl = mutableAclService.readAclById(dossierIdentity);
+        Set<String> members = new HashSet<>();
+        acl.getEntries()
+                .forEach(entry -> {
+                    if (entry.getSid() instanceof PrincipalSid) {
+                        var principal = ((PrincipalSid) entry.getSid()).getPrincipal();
+                        if (entry.getPermission().getMask() == RedPermission.OWNER.getMask()) {
+                            members.add(principal);
+                        }
+                    }
+                });
+        return members;
+    }
+
+
     @Override
     public String getIdentifier() {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java

@@ -1,92 +1,559 @@
 package com.iqser.red.service.persistence.management.v1.processor.configuration;
 
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
 
+import org.springframework.amqp.core.DirectExchange;
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+import com.knecon.fforesight.llm.service.QueueNames;
+
 import lombok.RequiredArgsConstructor;
 
 @Configuration
 @RequiredArgsConstructor
 public class MessagingConfiguration {
 
-    public static final String REDACTION_QUEUE = "redactionQueue";
-    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
-    public static final String REDACTION_ANALYSIS_RESPONSE_QUEUE = "redactionAnalysisResponseQueue";
-    public static final String REDACTION_DQL = "redactionDQL";
+    // persistence-service
+    public static final String DOWNLOAD_REQUEST_QUEUE_PREFIX = "download_request";
+    public static final String DOWNLOAD_REQUEST_EXCHANGE = "download_request_exchange";
+    public static final String DOWNLOAD_DLQ = "download_error";
 
-    public static final String DOWNLOAD_QUEUE = "downloadQueue";
-    public static final String DOWNLOAD_DLQ = "downloadDLQ";
+    public static final String DOWNLOAD_COMPRESSION_REQUEST_QUEUE_PREFIX = "download_compression_request";
+    public static final String DOWNLOAD_COMPRESSION_REQUEST_EXCHANGE = "download_compression_request_exchange";
+    public static final String DOWNLOAD_COMPRESSION_DLQ = "download_compression_error";
 
-    public static final String DOWNLOAD_COMPRESSION_QUEUE = "download_compression_queue";
-    public static final String DOWNLOAD_COMPRESSION_DLQ = "download_compression_dlq";
+    public static final String EXPORT_DOWNLOAD_REQUEST_QUEUE_PREFIX = "export_download_request";
+    public static final String EXPORT_DOWNLOAD_REQUEST_EXCHANGE = "export_download_request_exchange";
+    public static final String EXPORT_DOWNLOAD_DLQ = "export_download_error";
 
-    public static final String EXPORT_DOWNLOAD_QUEUE = "exportDownloadQueue";
-    public static final String EXPORT_DOWNLOAD_DLQ = "exportDownloadDLQ";
+    public static final String ANALYSIS_FLAG_CALCULATION_REQUEST_QUEUE_PREFIX = "analysis_flag_calculation_request";
+    public static final String ANALYSIS_FLAG_CALCULATION_REQUEST_EXCHANGE = "analysis_flag_calculation_request_exchange";
 
-    public static final String REPORT_QUEUE = "reportQueue";
-    public static final String REPORT_DLQ = "reportDLQ";
+    public static final String REDACTION_RESPONSE_QUEUE_PREFIX = "redaction_response";
+    public static final String REDACTION_RESPONSE_EXCHANGE = "redaction_response_exchange";
 
-    public static final String REPORT_RESULT_QUEUE = "reportResultQueue";
-    public static final String REPORT_RESULT_DLQ = "reportResultDLQ";
+    public static final String SEARCH_BULK_LOCAL_TERM_RESPONSE_QUEUE_PREFIX = "search_bulk_local_term_response";
+    public static final String SEARCH_BULK_LOCAL_TERM_RESPONSE_EXCHANGE = "search_bulk_local_term_response_exchange";
+    public static final String SEARCH_BULK_LOCAL_TERM_DLQ = "search_bulk_local_term_error";
 
-    public static final String OCR_REQUEST_QUEUE = "ocr_request_queue";
-    public static final String OCR_RESPONSE_QUEUE = "ocr_response_queue";
-    public static final String OCR_DLQ = "ocr_dead_letter_queue";
-    public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue";
-    public static final String OCR_STATUS_UPDATE_RESPONSE_DQL = "ocr_status_update_dead_letter_queue";
+    public static final String PDFTRON_RESPONSE_QUEUE_PREFIX = "pdftron_response";
+    public static final String PDFTRON_RESPONSE_EXCHANGE = "pdftron_response_exchange";
 
-    public static final String INDEXING_QUEUE = "indexingQueue";
-    public static final String INDEXING_DQL = "indexingDQL";
+    public static final String REPORT_RESPONSE_QUEUE_PREFIX = "report_response";
+    public static final String REPORT_RESPONSE_EXCHANGE = "report_response_exchange";
+    public static final String REPORT_RESPONSE_DLQ = "report_response_error";
 
-    public static final String DELETE_FROM_INDEX_QUEUE = "deleteFromIndexQueue";
-    public static final String DELETE_FROM_INDEX_DLQ = "deleteFromIndexDLQ";
+    public static final String OCR_RESPONSE_QUEUE_PREFIX = "ocr_response";
+    public static final String OCR_RESPONSE_EXCHANGE = "ocr_response_exchange";
+    public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE_PREFIX = "ocr_status_update_response";
+    public static final String OCR_STATUS_UPDATE_RESPONSE_EXCHANGE = "ocr_status_update_response_exchange";
+    public static final String OCR_STATUS_UPDATE_DLQ = "ocr_status_update_error";
 
-    public static final String IMAGE_SERVICE_QUEUE = "image_request_queue";
-    public static final String IMAGE_SERVICE_RESPONSE_QUEUE = "image_response_queue";
-    public static final String IMAGE_SERVICE_DLQ = "image_dead_letter_queue";
+    public static final String IMAGE_RESPONSE_QUEUE_PREFIX = "image_response";
+    public static final String IMAGE_RESPONSE_EXCHANGE = "image_response_exchange";
 
-    public static final String NER_SERVICE_QUEUE = "entity_request_queue";
-    public static final String NER_SERVICE_RESPONSE_QUEUE = "entity_response_queue";
-    public static final String NER_SERVICE_DLQ = "entity_dead_letter_queue";
+    public static final String ENTITY_RESPONSE_QUEUE_PREFIX = "entity_response";
+    public static final String ENTITY_RESPONSE_EXCHANGE = "entity_response_exchange";
 
-    public static final String PRE_PROCESSING_QUEUE = "preprocessingQueue";
-    public static final String PRE_PROCESSING_DLQ = "preprocessingDLQ";
+    public static final String AZURE_ENTITY_RESPONSE_QUEUE_PREFIX = "azure_entity_response";
+    public static final String AZURE_ENTITY_RESPONSE_EXCHANGE = "azure_entity_response_exchange";
 
-    public static final String PDFTRON_QUEUE = "pdftron_queue";
-    public static final String PDFTRON_DLQ = "pdftron_dlq";
-    public static final String PDFTRON_RESULT_QUEUE = "pdftron_result_queue";
+    public static final String CHUNKING_RESPONSE_QUEUE_PREFIX = "chunking_response";
+    public static final String CHUNKING_RESPONSE_EXCHANGE = "chunking_response_exchange";
+    public static final String CHUNKING_DLQ = "chunking_error";
 
-    public static final String CV_ANALYSIS_QUEUE = "cv_analysis_request_queue";
-    public static final String CV_ANALYSIS_RESPONSE_QUEUE = "cv_analysis_response_queue";
-    public static final String CV_ANALYSIS_DLQ = "cv_analysis_dead_letter_queue";
+    public static final String CV_ANALYSIS_RESPONSE_QUEUE_PREFIX = "cv_analysis_response";
+    public static final String CV_ANALYSIS_RESPONSE_EXCHANGE = "cv_analysis_response_exchange";
 
-    public static final String VISUAL_LAYOUT_PARSING_QUEUE = "visual_layout_parsing_service_queue";
-    public static final String VISUAL_LAYOUT_RESPONSE_QUEUE = "visual_layout_parsing_service_response_queue";
-    public static final String VISUAL_LAYOUT_DLQ = "visual_layout_parsing_service_dead_letter_queue";
-    public static final String ANALYSIS_FLAG_CALCULATION_QUEUE = "analysis_flag_calculation_queue";
+    public static final String VISUAL_LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX = "visual_layout_parsing_response";
+    public static final String VISUAL_LAYOUT_PARSING_RESPONSE_EXCHANGE = "visual_layout_parsing_response_exchange";
+
+    // pdftron-redaction-service
+    public static final String PREPROCESSING_REQUEST_EXCHANGE = "preprocessing_request_exchange";
+    public static final String PREPROCESSING_DLQ = "preprocessing_error";
+
+    public static final String PDFTRON_REQUEST_EXCHANGE = "pdftron_request_exchange";
+    public static final String PDFTRON_DLQ = "pdftron_error";
+
+    // redaction-service
+    public static final String REDACTION_REQUEST_QUEUE_PREFIX = "redaction_request";
+    public static final String REDACTION_REQUEST_EXCHANGE = "redaction_request_exchange";
+    public static final String REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX = "redaction_priority_request";
+    public static final String REDACTION_PRIORITY_REQUEST_EXCHANGE = "redaction_priority_request_exchange";
+    public static final String REDACTION_DLQ = "redaction_error";
+
+    // redaction-report-service
+    public static final String REPORT_REQUEST_EXCHANGE = "report_request_exchange";
+    public static final String REPORT_REQUEST_DLQ = "report_request_error";
+
+    // search-service
+    public static final String INDEXING_REQUEST_EXCHANGE = "indexing_request_exchange";
+    public static final String INDEXING_DLQ = "indexing_error";
+
+    public static final String DELETE_FROM_INDEX_REQUEST_EXCHANGE = "delete_from_index_request_exchange";
+    public static final String DELETE_FROM_INDEX_DLQ = "delete_from_index_error";
+
+    // ocr-services
+    public static final String OCR_REQUEST_EXCHANGE = "ocr_request_exchange";
+    public static final String OCR_DLQ = "ocr_error";
+
+    // image-service
+    public static final String IMAGE_REQUEST_EXCHANGE = "image_request_exchange";
+    public static final String IMAGE_DLQ = "image_error";
+
+    // entity-recognition-service
+    public static final String ENTITY_REQUEST_EXCHANGE = "entity_request_exchange";
+    public static final String ENTITY_DLQ = "entity_error";
+
+    // azure-ner-service
+    public static final String AZURE_ENTITY_REQUEST_EXCHANGE = "azure_entity_request_exchange";
+    public static final String AZURE_ENTITY_DLQ = "azure_entity_error";
+
+    // cv-analysis-service
+    public static final String CV_ANALYSIS_REQUEST_EXCHANGE = "cv_analysis_request_exchange";
+    public static final String CV_ANALYSIS_DLQ = "cv_analysis_error";
+
+    // visual-layout-parsing-service
+    public static final String VISUAL_LAYOUT_PARSING_REQUEST_EXCHANGE = "visual_layout_parsing_request_exchange";
+    public static final String VISUAL_LAYOUT_PARSING_DLQ = "visual_layout_parsing_error";
+
+    //chunking-service
+    public static final String CHUNKING_REQUEST_EXCHANGE = "chunking_request_exchange";
+
+    //llm-ner-service
+    public static final String LLM_NER_REQUEST_EXCHANGE = QueueNames.LLM_NER_REQUEST_EXCHANGE;
+    public static final String LLM_NER_RESPONSE_QUEUE_PREFIX = QueueNames.LLM_NER_RESPONSE_QUEUE_PREFIX;
+    public static final String LLM_NER_RESPONSE_EXCHANGE = QueueNames.LLM_NER_RESPONSE_EXCHANGE;
+    public static final String LLM_NER_DLQ = QueueNames.LLM_NER_DLQ;
 
     public static final String X_ERROR_INFO_HEADER = "x-error-message";
     public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
 
-
-    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_dead_letter_queue";
-
-    // --- Saas Migration, can be removed later ----
-
-    public static final String MIGRATION_QUEUE = "migrationQueue";
-    public static final String MIGRATION_DLQ = "migrationDLQ";
+    // ---- Saas Migration, can be removed later ----
+    public static final String MIGRATION_REQUEST_QUEUE = "migrationQueue";
     public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
+    public static final String MIGRATION_DLQ = "migrationDLQ";
+
+
+    // ---- persistence-service ----
+    @Bean
+    public DirectExchange downloadExchange() {
+
+        return new DirectExchange(DOWNLOAD_REQUEST_EXCHANGE);
+    }
 
 
+    @Bean
+    public Queue downloadDeadLetterQueue() {
+
+        return QueueBuilder.durable(DOWNLOAD_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange downloadCompressionExchange() {
+
+        return new DirectExchange(DOWNLOAD_COMPRESSION_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue downloadCompressionQueueDLQ() {
+
+        return QueueBuilder.durable(DOWNLOAD_COMPRESSION_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange exportDownloadExchange() {
+
+        return new DirectExchange(EXPORT_DOWNLOAD_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue exportDownloadDeadLetterQueue() {
+
+        return QueueBuilder.durable(EXPORT_DOWNLOAD_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange analysisFlagCalculationExchange() {
+
+        return new DirectExchange(ANALYSIS_FLAG_CALCULATION_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange redactionResponseExchange() {
+
+        return new DirectExchange(REDACTION_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange searchTermOccurrencesResponseExchange() {
+
+        return new DirectExchange(SEARCH_BULK_LOCAL_TERM_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue searchTermOccurrencesDLQ() {
+
+        return QueueBuilder.durable(SEARCH_BULK_LOCAL_TERM_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange reportResponseExchange() {
+
+        return new DirectExchange(REPORT_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange chunkingResponseExchange() {
+
+        return new DirectExchange(CHUNKING_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue chunkingDLQ() {
+
+        return QueueBuilder.durable(CHUNKING_DLQ).build();
+    }
+
+
+    @Bean
+    public Queue reportResponseDLQ() {
+
+        return QueueBuilder.durable(REPORT_RESPONSE_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange ocrResponseExchange() {
+
+        return new DirectExchange(OCR_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange imageResponseExchange() {
+
+        return new DirectExchange(IMAGE_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange entityResponseExchange() {
+
+        return new DirectExchange(ENTITY_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange azureEntityResponseExchange() {
+
+        return new DirectExchange(AZURE_ENTITY_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange cvAnalysisResponseExchange() {
+
+        return new DirectExchange(CV_ANALYSIS_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange visualLayoutParsingResponseExchange() {
+
+        return new DirectExchange(VISUAL_LAYOUT_PARSING_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange layoutParsingResponseExchange() {
+
+        return new DirectExchange(LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE);
+    }
+
+
+    // ---- pdftron-redaction-service ----
+    @Bean
+    public DirectExchange preprocessingRequestExchange() {
+
+        return new DirectExchange(PREPROCESSING_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange preprocessingResponseExchange() {
+
+        return new DirectExchange(PREPROCESSING_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue preprocessingDLQ() {
+
+        return QueueBuilder.durable(PREPROCESSING_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange pdftronRequestExchange() {
+
+        return new DirectExchange(PDFTRON_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange pdftronResponseExchange() {
+
+        return new DirectExchange(PDFTRON_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue pdfTronDLQ() {
+
+        return QueueBuilder.durable(PDFTRON_DLQ).build();
+    }
+
+
+    // ---- redaction-service ----
+    @Bean
+    public DirectExchange redactionRequestExchange() {
+
+        return new DirectExchange(REDACTION_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange redactionPriorityRequestExchange() {
+
+        return new DirectExchange(REDACTION_PRIORITY_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue redactionDLQ() {
+
+        return QueueBuilder.durable(REDACTION_DLQ).build();
+    }
+
+
+    // ---- redaction-report-service ----
+    @Bean
+    public DirectExchange reportRequestExchange() {
+
+        return new DirectExchange(REPORT_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue reportRequestDLQ() {
+
+        return QueueBuilder.durable(REPORT_REQUEST_DLQ).build();
+    }
+
+
+    // ---- search-service ----
+    @Bean
+    public DirectExchange indexingRequestExchange() {
+
+        return new DirectExchange(INDEXING_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue indexingDLQ() {
+
+        return QueueBuilder.durable(INDEXING_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange deleteFromIndexRequestExchange() {
+
+        return new DirectExchange(DELETE_FROM_INDEX_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue deleteFromIndexDLQ() {
+
+        return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build();
+    }
+
+
+    // ---- ocr-services ----
+    @Bean
+    public DirectExchange ocrRequestExchange() {
+
+        return new DirectExchange(OCR_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue ocrDLQ() {
+
+        return QueueBuilder.durable(OCR_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange ocrStatusUpdateResponseExchange() {
+
+        return new DirectExchange(OCR_STATUS_UPDATE_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue ocrStatusUpdateDLQ() {
+
+        return QueueBuilder.durable(OCR_STATUS_UPDATE_DLQ).build();
+    }
+
+
+    // ---- image-service ----
+    @Bean
+    public DirectExchange imageRequestExchange() {
+
+        return new DirectExchange(IMAGE_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue imageDLQ() {
+
+        return QueueBuilder.durable(IMAGE_DLQ).build();
+    }
+
+
+    // ---- entity-recognition-service ----
+    @Bean
+    public DirectExchange nerRequestExchange() {
+
+        return new DirectExchange(ENTITY_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue nerDLQ() {
+
+        return QueueBuilder.durable(ENTITY_DLQ).build();
+    }
+
+
+    // ---- azure-ner-service ----
+    @Bean
+    public DirectExchange azureNerRequestExchange() {
+
+        return new DirectExchange(AZURE_ENTITY_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue azureNerDLQ() {
+
+        return QueueBuilder.durable(AZURE_ENTITY_DLQ).build();
+    }
+
+
+    // ---- cv-analysis-service ----
+    @Bean
+    public DirectExchange cvAnalysisRequestExchange() {
+
+        return new DirectExchange(CV_ANALYSIS_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue cvAnalysisDLQ() {
+
+        return QueueBuilder.durable(CV_ANALYSIS_DLQ).build();
+    }
+
+
+    // ---- visual-layout-parsing-service ----
+    @Bean
+    public DirectExchange visualLayoutParsingRequestExchange() {
+
+        return new DirectExchange(VISUAL_LAYOUT_PARSING_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue visualLayoutParsingDLQ() {
+
+        return QueueBuilder.durable(VISUAL_LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    // ---- layoutparser-service ----
+    @Bean
+    public DirectExchange layoutParsingRequestExchange() {
+
+        return new DirectExchange(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue layoutParsingDLQ() {
+
+        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    // ---- chunking-service ----
+    @Bean
+    public DirectExchange chunkingRequestExchange() {
+
+        return new DirectExchange(CHUNKING_REQUEST_EXCHANGE);
+    }
+
+
+    // ----- llm-ner-service ----
+    @Bean
+    public DirectExchange llmNerRequestExchange() {
+
+        return new DirectExchange(LLM_NER_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange llmNerResponseExchange() {
+
+        return new DirectExchange(LLM_NER_RESPONSE_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue llmNerDLQ() {
+
+        return QueueBuilder.durable(LLM_NER_DLQ).build();
+    }
+
+
+    // ---- Saas Migration ----
     @Bean
     public Queue migrationQueue() {
 
-        return QueueBuilder.durable(MIGRATION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
+        return QueueBuilder.durable(MIGRATION_REQUEST_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", MIGRATION_DLQ)
+                .maxPriority(2)
+                .build();
     }
 
 
@@ -107,352 +574,4 @@ public class MessagingConfiguration {
                 .build();
     }
 
-    // --- End Saas Migration
-
-
-    @Bean
-    public Queue nerRequestQueue() {
-
-        return QueueBuilder.durable(NER_SERVICE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue nerResponseQueue() {
-
-        return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue nerResponseDLQ() {
-
-        return QueueBuilder.durable(NER_SERVICE_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue imageRequestQueue() {
-
-        return QueueBuilder.durable(IMAGE_SERVICE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue imageResponseQueue() {
-
-        return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue imageResponseDLQ() {
-
-        return QueueBuilder.durable(IMAGE_SERVICE_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue cvAnalysisRequestQueue() {
-
-        return QueueBuilder.durable(CV_ANALYSIS_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", CV_ANALYSIS_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue cvAnalysisResponseQueue() {
-
-        return QueueBuilder.durable(CV_ANALYSIS_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", CV_ANALYSIS_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue cvAnalysisResponseDLQ() {
-
-        return QueueBuilder.durable(CV_ANALYSIS_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue visualLayoutParsingRequestQueue() {
-
-        return QueueBuilder.durable(VISUAL_LAYOUT_PARSING_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", VISUAL_LAYOUT_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue visualLayoutParsingResponseQueue() {
-
-        return QueueBuilder.durable(VISUAL_LAYOUT_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", VISUAL_LAYOUT_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue visualLayoutParsingDLQ() {
-
-        return QueueBuilder.durable(VISUAL_LAYOUT_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue ocrStatusUpdateResponseQueue() {
-
-        return QueueBuilder.durable(OCR_STATUS_UPDATE_RESPONSE_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", OCR_STATUS_UPDATE_RESPONSE_DQL)
-                .build();
-    }
-
-
-    @Bean
-    public Queue ocrStatusUpdateResponseDQL() {
-
-        return QueueBuilder.durable(OCR_STATUS_UPDATE_RESPONSE_DQL).build();
-    }
-
-
-    @Bean
-    public Queue redactionQueue() {
-
-        return QueueBuilder.durable(REDACTION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REDACTION_DQL).maxPriority(2).build();
-    }
-
-
-    @Bean
-    public Queue redactionPriorityQueue() {
-
-        return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue redactionAnalysisResponseQueue() {
-
-        return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue ocrRequestQueue() {
-
-        return QueueBuilder.durable(OCR_REQUEST_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", OCR_DLQ)
-                .withArgument("x-max-priority", 2) // Higher value is higher priority.
-                .maxPriority(2)
-                .build();
-    }
-
-    @Bean
-    public Queue ocrResponseQueue() {
-
-        return QueueBuilder.durable(OCR_RESPONSE_QUEUE)
-                .build();
-    }
-
-
-    @Bean
-    public Queue ocrDLQ() {
-
-        return QueueBuilder.durable(OCR_DLQ)
-                .build();
-    }
-
-
-    @Bean
-    public Queue downloadQueue() {
-
-        return QueueBuilder.durable(DOWNLOAD_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", DOWNLOAD_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue downloadDeadLetterQueue() {
-
-        return QueueBuilder.durable(DOWNLOAD_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue downloadCompressionQueue() {
-
-        return QueueBuilder.durable(DOWNLOAD_COMPRESSION_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", DOWNLOAD_COMPRESSION_DLQ)
-                .build();
-    }
-
-
-    @Bean
-    public Queue downloadCompressionQueueDLQ() {
-
-        return QueueBuilder.durable(DOWNLOAD_COMPRESSION_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue exportDownloadQueue() {
-
-        return QueueBuilder.durable(EXPORT_DOWNLOAD_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", EXPORT_DOWNLOAD_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue exportDownloadDeadLetterQueue() {
-
-        return QueueBuilder.durable(EXPORT_DOWNLOAD_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue reportQueue() {
-
-        return QueueBuilder.durable(REPORT_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", REPORT_DLQ)
-                .withArgument("x-max-priority", 2)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue reportDeadLetterQueue() {
-
-        return QueueBuilder.durable(REPORT_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue reportResultQueue() {
-
-        return QueueBuilder.durable(REPORT_RESULT_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue reportResultDeadLetterQueue() {
-
-        return QueueBuilder.durable(REPORT_RESULT_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue analysisFlagCalculationQueue() {
-
-        return QueueBuilder.durable(ANALYSIS_FLAG_CALCULATION_QUEUE).build();
-    }
-
-
-    @Bean
-    public Queue indexingQueue() {
-
-        return QueueBuilder.durable(INDEXING_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", INDEXING_DQL).maxPriority(2).build();
-    }
-
-
-    @Bean
-    public Queue indexingDeadLetterQueue() {
-
-        return QueueBuilder.durable(INDEXING_DQL).build();
-    }
-
-
-    @Bean
-    public Queue deleteFromIndexQueue() {
-
-        return QueueBuilder.durable(DELETE_FROM_INDEX_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", DELETE_FROM_INDEX_DLQ)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue deleteFromIndexDLQ() {
-
-        return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue preprocessingQueue() {
-
-        return QueueBuilder.durable(PRE_PROCESSING_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PRE_PROCESSING_DLQ)
-                .withArgument("x-max-priority", 2)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue preprocessingDeadLetterQueue() {
-
-        return QueueBuilder.durable(PRE_PROCESSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue pdfTronQueue() {
-
-        return QueueBuilder.durable(PDFTRON_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PDFTRON_DLQ)
-                .withArgument("x-max-priority", 2)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue pdfTronDlq() {
-
-        return QueueBuilder.durable(PDFTRON_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue pdfTronResultQueue() {
-
-        return QueueBuilder.durable(PDFTRON_RESULT_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PDFTRON_DLQ)
-                .withArgument("x-max-priority", 2)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingRequestQueue() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingResponseQueue() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingDLQ() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
-    }
-
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/QueueProviderConfiguration.java

@@ -0,0 +1,198 @@
+package com.iqser.red.service.persistence.management.v1.processor.configuration;
+
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.ANALYSIS_FLAG_CALCULATION_REQUEST_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.ANALYSIS_FLAG_CALCULATION_REQUEST_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.AZURE_ENTITY_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.AZURE_ENTITY_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.AZURE_ENTITY_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.CHUNKING_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.CHUNKING_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.CHUNKING_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.CV_ANALYSIS_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.CV_ANALYSIS_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.DOWNLOAD_COMPRESSION_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.DOWNLOAD_COMPRESSION_REQUEST_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.DOWNLOAD_COMPRESSION_REQUEST_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.DOWNLOAD_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.DOWNLOAD_REQUEST_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.DOWNLOAD_REQUEST_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.ENTITY_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.ENTITY_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.ENTITY_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.EXPORT_DOWNLOAD_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.EXPORT_DOWNLOAD_REQUEST_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.EXPORT_DOWNLOAD_REQUEST_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.IMAGE_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.IMAGE_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.IMAGE_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.LLM_NER_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.LLM_NER_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.LLM_NER_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_STATUS_UPDATE_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.PDFTRON_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.PDFTRON_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.PDFTRON_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.REDACTION_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.REDACTION_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.REDACTION_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.REPORT_RESPONSE_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.REPORT_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.REPORT_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.SEARCH_BULK_LOCAL_TERM_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.SEARCH_BULK_LOCAL_TERM_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.SEARCH_BULK_LOCAL_TERM_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.VISUAL_LAYOUT_PARSING_DLQ;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.VISUAL_LAYOUT_PARSING_RESPONSE_EXCHANGE;
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.VISUAL_LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX;
+import static com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessageReceiver.EXPORT_DOWNLOAD_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadCompressionMessageReceiver.DOWNLOAD_COMPRESSION_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadMessageReceiver.DOWNLOAD_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadReportMessageReceiver.REPORT_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.download.RedactionResultMessageReceiver.PDFTRON_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.AnalysisFlagsCalculationMessageReceiver.ANALYSIS_FLAG_CALCULATION_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.AzureNerMessageReceiver.AZURE_ENTITY_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.ChunkingMessageReceiver.CHUNKING_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.CvAnalysisMessageReceiver.CV_ANALYSIS_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.ImageMessageReceiver.IMAGE_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.LayoutParsingFinishedMessageReceiver.LAYOUT_PARSING_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.NerMessageReceiver.ENTITY_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.NerMessageReceiver.LLM_ENTITY_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.OCRProcessingMessageReceiver.OCR_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.OCRProcessingMessageReceiver.OCR_STATUS_UPDATE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.RedactionAnalysisResponseReceiver.REDACTION_RESPONSE_LISTENER_ID;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.SearchTermOccurrencesResponseReceiver.SEARCH_BULK_LOCAL_TERM_RESPONSE_LISTENER;
+import static com.iqser.red.service.persistence.management.v1.processor.service.queue.VisualLayoutParsingMessageReceiver.VISUAL_LAYOUT_PARSING_RESPONSE_LISTENER_ID;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
+import com.knecon.fforesight.tenantcommons.model.TenantQueueProvider;
+
+@Configuration
+public class QueueProviderConfiguration {
+
+    @Bean
+    public TenantQueueProvider tenantQueueProvider() {
+
+        var queues = Set.of(TenantQueueConfiguration.builder()
+                        .listenerId(DOWNLOAD_LISTENER_ID)
+                        .exchangeName(DOWNLOAD_REQUEST_EXCHANGE)
+                        .queuePrefix(DOWNLOAD_REQUEST_QUEUE_PREFIX)
+                        .dlqName(DOWNLOAD_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(DOWNLOAD_COMPRESSION_LISTENER_ID)
+                        .exchangeName(DOWNLOAD_COMPRESSION_REQUEST_EXCHANGE)
+                        .queuePrefix(DOWNLOAD_COMPRESSION_REQUEST_QUEUE_PREFIX)
+                        .dlqName(DOWNLOAD_COMPRESSION_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(EXPORT_DOWNLOAD_LISTENER_ID)
+                        .exchangeName(EXPORT_DOWNLOAD_REQUEST_EXCHANGE)
+                        .queuePrefix(EXPORT_DOWNLOAD_REQUEST_QUEUE_PREFIX)
+                        .dlqName(EXPORT_DOWNLOAD_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(ANALYSIS_FLAG_CALCULATION_LISTENER_ID)
+                        .exchangeName(ANALYSIS_FLAG_CALCULATION_REQUEST_EXCHANGE)
+                        .queuePrefix(ANALYSIS_FLAG_CALCULATION_REQUEST_QUEUE_PREFIX)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(REDACTION_RESPONSE_LISTENER_ID)
+                        .exchangeName(REDACTION_RESPONSE_EXCHANGE)
+                        .queuePrefix(REDACTION_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(REDACTION_DLQ)
+                        .arguments(Map.of("x-max-priority", 2))
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(SEARCH_BULK_LOCAL_TERM_RESPONSE_LISTENER)
+                        .exchangeName(SEARCH_BULK_LOCAL_TERM_RESPONSE_EXCHANGE)
+                        .queuePrefix(SEARCH_BULK_LOCAL_TERM_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(SEARCH_BULK_LOCAL_TERM_DLQ)
+                        .arguments(Map.of("x-max-priority", 2))
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(PDFTRON_RESPONSE_LISTENER_ID)
+                        .exchangeName(PDFTRON_RESPONSE_EXCHANGE)
+                        .queuePrefix(PDFTRON_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(PDFTRON_DLQ)
+                        .arguments(Map.of("x-max-priority", 2))
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(REPORT_RESPONSE_LISTENER_ID)
+                        .exchangeName(REPORT_RESPONSE_EXCHANGE)
+                        .queuePrefix(REPORT_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(REPORT_RESPONSE_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(OCR_RESPONSE_LISTENER_ID)
+                        .exchangeName(OCR_RESPONSE_EXCHANGE)
+                        .queuePrefix(OCR_RESPONSE_QUEUE_PREFIX)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(OCR_STATUS_UPDATE_LISTENER_ID)
+                        .exchangeName(OCR_STATUS_UPDATE_RESPONSE_EXCHANGE)
+                        .queuePrefix(OCR_STATUS_UPDATE_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(OCR_STATUS_UPDATE_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(IMAGE_RESPONSE_LISTENER_ID)
+                        .exchangeName(IMAGE_RESPONSE_EXCHANGE)
+                        .queuePrefix(IMAGE_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(IMAGE_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(ENTITY_RESPONSE_LISTENER_ID)
+                        .exchangeName(ENTITY_RESPONSE_EXCHANGE)
+                        .queuePrefix(ENTITY_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(ENTITY_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(AZURE_ENTITY_RESPONSE_LISTENER_ID)
+                        .exchangeName(AZURE_ENTITY_RESPONSE_EXCHANGE)
+                        .queuePrefix(AZURE_ENTITY_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(AZURE_ENTITY_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(CHUNKING_RESPONSE_LISTENER_ID)
+                        .exchangeName(CHUNKING_RESPONSE_EXCHANGE)
+                        .queuePrefix(CHUNKING_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(CHUNKING_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(LLM_ENTITY_RESPONSE_LISTENER_ID)
+                        .exchangeName(LLM_NER_RESPONSE_EXCHANGE)
+                        .queuePrefix(LLM_NER_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(LLM_NER_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(CV_ANALYSIS_RESPONSE_LISTENER_ID)
+                        .exchangeName(CV_ANALYSIS_RESPONSE_EXCHANGE)
+                        .queuePrefix(CV_ANALYSIS_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(CV_ANALYSIS_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(VISUAL_LAYOUT_PARSING_RESPONSE_LISTENER_ID)
+                        .exchangeName(VISUAL_LAYOUT_PARSING_RESPONSE_EXCHANGE)
+                        .queuePrefix(VISUAL_LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX)
+                        .dlqName(VISUAL_LAYOUT_PARSING_DLQ)
+                        .build(),
+                TenantQueueConfiguration.builder()
+                        .listenerId(LAYOUT_PARSING_RESPONSE_LISTENER_ID)
+                        .exchangeName(LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE)
+                        .queuePrefix(LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX)
+                        .build());
+        return new TenantQueueProvider(queues);
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/UserMessagingConfiguration.java

@@ -16,13 +16,13 @@ import lombok.RequiredArgsConstructor;
 @RequiredArgsConstructor
 public class UserMessagingConfiguration {
 
-    public static final String PERSISTENCE_SERVICE_USER_CREATED_QUEUE = "persistence-service-user-created-queue";
-    public static final String PERSISTENCE_SERVICE_USER_DELETED_QUEUE = "persistence-service-user-deleted-queue";
-    public static final String PERSISTENCE_SERVICE_USER_UPDATED_QUEUE = "persistence-service-user-updated-queue";
-    public static final String PERSISTENCE_SERVICE_USER_STATUS_CHANGED_QUEUE = "persistence-service-user-status-changed-queue";
-    public static final String PERSISTENCE_SERVICE_USER_ROLES_UPDATED_QUEUE = "persistence-service-user-roles-updated-queue";
-    public static final String PERSISTENCE_SERVICE_USER_OWN_PROFILE_UPDATED_QUEUE = "persistence-service-user-own-profile-updated-queue";
-    public static final String PERSISTENCE_SERVICE_USER_EVENTS_DQL = "persistence-service-user-events-dql";
+    public static final String PERSISTENCE_SERVICE_USER_CREATED_QUEUE = "persistence-service-user-created";
+    public static final String PERSISTENCE_SERVICE_USER_DELETED_QUEUE = "persistence-service-user-deleted";
+    public static final String PERSISTENCE_SERVICE_USER_UPDATED_QUEUE = "persistence-service-user-updated";
+    public static final String PERSISTENCE_SERVICE_USER_STATUS_CHANGED_QUEUE = "persistence-service-user-status-changed";
+    public static final String PERSISTENCE_SERVICE_USER_ROLES_UPDATED_QUEUE = "persistence-service-user-roles-updated";
+    public static final String PERSISTENCE_SERVICE_USER_OWN_PROFILE_UPDATED_QUEUE = "persistence-service-user-own-profile-updated";
+    public static final String PERSISTENCE_SERVICE_USER_EVENTS_DLQ = "persistence-service-user-events-error";
 
 
     @Bean("persistenceServiceUserRolesUpdatedQueue")
@@ -30,7 +30,7 @@ public class UserMessagingConfiguration {
 
         return QueueBuilder.durable(PERSISTENCE_SERVICE_USER_ROLES_UPDATED_QUEUE)
                 .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DQL)
+                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DLQ)
                 .build();
     }
 
@@ -48,7 +48,7 @@ public class UserMessagingConfiguration {
 
         return QueueBuilder.durable(PERSISTENCE_SERVICE_USER_STATUS_CHANGED_QUEUE)
                 .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DQL)
+                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DLQ)
                 .build();
     }
 
@@ -66,7 +66,7 @@ public class UserMessagingConfiguration {
 
         return QueueBuilder.durable(PERSISTENCE_SERVICE_USER_UPDATED_QUEUE)
                 .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DQL)
+                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DLQ)
                 .build();
     }
 
@@ -84,7 +84,7 @@ public class UserMessagingConfiguration {
 
         return QueueBuilder.durable(PERSISTENCE_SERVICE_USER_DELETED_QUEUE)
                 .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DQL)
+                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DLQ)
                 .build();
     }
 
@@ -102,7 +102,7 @@ public class UserMessagingConfiguration {
 
         return QueueBuilder.durable(PERSISTENCE_SERVICE_USER_OWN_PROFILE_UPDATED_QUEUE)
                 .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DQL)
+                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DLQ)
                 .build();
     }
 
@@ -120,7 +120,7 @@ public class UserMessagingConfiguration {
 
         return QueueBuilder.durable(PERSISTENCE_SERVICE_USER_CREATED_QUEUE)
                 .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DQL)
+                .withArgument("x-dead-letter-routing-key", PERSISTENCE_SERVICE_USER_EVENTS_DLQ)
                 .build();
     }
 
@@ -136,7 +136,7 @@ public class UserMessagingConfiguration {
     @Bean
     public Queue persistenceServiceUserEventsDLQ() {
 
-        return QueueBuilder.durable(PERSISTENCE_SERVICE_USER_EVENTS_DQL).build();
+        return QueueBuilder.durable(PERSISTENCE_SERVICE_USER_EVENTS_DLQ).build();
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/ExportDownloadMessageReceiver.java

@@ -28,6 +28,7 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ExportDownloadMessageReceiver {
 
+    public static final String EXPORT_DOWNLOAD_LISTENER_ID = "export-download-listener";
     static long MB = 1024 * 1024;
     DossierTemplateExportService dossierTemplateService;
     FileExchangeExportService fileExchangeExportService;
@@ -36,7 +37,7 @@ public class ExportDownloadMessageReceiver {
 
 
     @RabbitHandler
-    @RabbitListener(queues = MessagingConfiguration.EXPORT_DOWNLOAD_QUEUE)
+    @RabbitListener(id = EXPORT_DOWNLOAD_LISTENER_ID)
     public void receive(ExportDownloadMessage downloadJob) throws JsonProcessingException {
 
         var download = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId());

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/ExportFilename.java

@@ -14,6 +14,7 @@ public enum ExportFilename {
     LEGAL_BASIS("legalBasisMappingList"),
     RULES("rules"),
     COMPONENT_RULES("componentRules"),
+    DATE_FORMATS("dateFormats"),
     REPORT_TEMPLATE("reportTemplateList"),
     REPORT_TEMPLATE_MULTI_FILE(" (Multi-file)"),
     DOSSIER_TYPE("dossierType"),

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java

@@ -22,10 +22,13 @@ public class FileExchangeNames {
 
     public static Definition STRUCTURE = new Definition(FileType.DOCUMENT_STRUCTURE);
     public static Definition PAGES = new Definition(FileType.DOCUMENT_PAGES);
+    public static Definition CHUNKS = new Definition(FileType.DOCUMENT_CHUNKS);
     public static Definition TEXT = new Definition(FileType.DOCUMENT_TEXT);
     public static Definition POSITIONS = new Definition(FileType.DOCUMENT_POSITION);
     public static Definition SIMPLIFIED_TEXT = new Definition(FileType.SIMPLIFIED_TEXT);
     public static Definition NER_ENTITIES = new Definition(FileType.NER_ENTITIES);
+    public static Definition LLM_NER_ENTITIES = new Definition(FileType.LLM_NER_ENTITIES);
+    public static Definition AZURE_NER_ENTITIES = new Definition(FileType.AZURE_NER_ENTITIES);
 
     public static Definition TABLES = new Definition(FileType.TABLES);
     public static Definition IMAGES = new Definition(FileType.IMAGE_INFO);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java

@@ -0,0 +1,57 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DatasetExchangeImportModel {
+
+    private ImportTemplateResult importTemplateResult = ImportTemplateResult.builder().build();
+    private List<Dossier> dossiers = new LinkedList<>();
+
+    @Builder
+    @Getter
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    public static class Dossier {
+
+        String name;
+        @Builder.Default
+        List<File> files = new LinkedList<>();
+
+    }
+
+    @Getter
+    @AllArgsConstructor
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    public static class File {
+
+        String name;
+        byte[] data;
+
+    }
+
+
+    public DatasetExchangeImportModel.Dossier getOrCreateDossier(String name) {
+
+        for (Dossier dossier : dossiers) {
+            if (dossier.getName().equals(name)) {
+                return dossier;
+            }
+        }
+        Dossier dossier = Dossier.builder().name(name).build();
+        dossiers.add(dossier);
+        return dossier;
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DateFormatsExportModel.java

@@ -0,0 +1,16 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DateFormatsExportModel {
+
+    long version;
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/FileExchangeImportModel.java

@@ -50,7 +50,8 @@ public class FileExchangeImportModel {
         EntityLog entityLog;
         ComponentLog componentLog;
         List<ComponentLogEntry> overrides;
-        ManualChangesExportModel manualChanges;
+        @Builder.Default
+        ManualChangesExportModel manualChanges = new ManualChangesExportModel();
         @Builder.Default
         List<FileAttribute> fileAttributes = new ArrayList<>();
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/ImportTemplateResult.java

@@ -62,6 +62,10 @@ public class ImportTemplateResult {
 
     public RulesExportModel componentRulesExportModel = new RulesExportModel();
 
+    public String dateFormats;
+
+    public DateFormatsExportModel dateFormatsExportModel = new DateFormatsExportModel();
+
     @Builder.Default
     public List<ReportTemplateUploadRequest> reportTemplateUploadRequests = new ArrayList<>();
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualAddExportModel.java

@@ -11,13 +11,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualChangesExportModel.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import lombok.AccessLevel;
@@ -16,11 +17,17 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class ManualChangesExportModel {
 
-    List<ManualAddExportModel> manualAddExportModels;
-    List<ManualRecategorizationExportModel> manualRecategorizationExportModels;
-    List<ManualResizeExportModel> manualResizeExportModels;
-    List<ManualForceExportModel> manualForceExportModels;
-    List<ManualLegalBasisChangeExportModel> manualLegalBasisChangeExportModels;
-    List<ManualRemoveExportModel> manualRemoveExportModels;
+    @Builder.Default
+    List<ManualAddExportModel> manualAddExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualRecategorizationExportModel> manualRecategorizationExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualResizeExportModel> manualResizeExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualForceExportModel> manualForceExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualLegalBasisChangeExportModel> manualLegalBasisChangeExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualRemoveExportModel> manualRemoveExportModels = new ArrayList<>();
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualForceExportModel.java

@@ -10,7 +10,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualLegalBasisChangeExportModel.java

@@ -8,7 +8,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class ManualLegalBasisChangeExportModel {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualRecategorizationExportModel.java

@@ -12,7 +12,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualRemoveExportModel.java

@@ -10,7 +10,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class ManualRemoveExportModel {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualResizeExportModel.java

@@ -14,7 +14,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java

@@ -0,0 +1,183 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import static com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeArchivalService.SIZE_THRESHOLD;
+
+import java.time.OffsetDateTime;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
+import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
+import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DatasetExchangeArchiveReader;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
+import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
+import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadPreparationService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DatasetExchangeService {
+
+    DossierTemplateManagementService dossierTemplateManagementService;
+    DownloadStatusPersistenceService downloadStatusPersistenceService;
+    FileStatusPersistenceService fileStatusPersistenceService;
+    DossierPersistenceService dossierPersistenceService;
+    ColorsService colorsService;
+    RabbitTemplate rabbitTemplate;
+    FileManagementServiceSettings settings;
+    DossierTemplateImportService dossierTemplateImportService;
+    DossierCreatorService dossierCreatorService;
+    ReportTemplatePersistenceService reportTemplateService;
+    UploadService uploadService;
+    DownloadPreparationService downloadPreparationService;
+    FileManagementStorageService storageService;
+
+
+    public DownloadResponse prepareExport(String dossierTemplateId) {
+
+        var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId);
+        String downloadFilename = template.getName() + "_PREVIEW_FILES.zip";
+        String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES");
+        List<FileEntity> files = fileStatusPersistenceService.getActiveFilesForDossierTemplate(dossierTemplateId);
+        List<DossierEntity> dossiers = dossierPersistenceService.findAllActiveDossiersForDossierTemplateId(dossierTemplateId);
+        if (dossiers.isEmpty()) {
+            throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId);
+        }
+        ColorsEntity colors = colorsService.getColors(dossierTemplateId);
+
+        downloadPreparationService.clearRedactionStatusEntries(storageId);
+        if (storageService.objectExists(storageId)) {
+            storageService.deleteObject(storageId);
+        }
+
+        DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.createStatus(KeycloakSecurity.getUserId(),
+                                                                                            storageId,
+                                                                                            dossiers.get(0),
+                                                                                            downloadFilename,
+                                                                                            "application/zip",
+                                                                                            files.stream()
+                                                                                                    .map(FileEntity::getId)
+                                                                                                    .toList(),
+                                                                                            Set.of(DownloadFileType.DATASET_EXPORT),
+                                                                                            Collections.emptyList(),
+                                                                                            colors.getPreviewColor());
+
+        RedactionMessage.RedactionMessageBuilder builder = RedactionMessage.builder()
+                .appliedRedactionColor(colors.getRedactionColor())
+                .redactionPreviewColor(colors.getPreviewColor())
+                .keepHiddenText(true)
+                .keepOverlappingObjects(true)
+                .keepImageMetaData(true)
+                .downloadId(storageId)
+                .redactionTypes(List.of(RedactionType.PREVIEW));
+
+        downloadStatus.getFiles()
+                .forEach(fileEntity -> {
+                    RedactionMessage message = builder.dossierId(fileEntity.getDossierId())
+                            .fileId(fileEntity.getId())
+                            .unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED)
+                            .build();
+                    log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", storageId, fileEntity.getId());
+                    rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_REQUEST_EXCHANGE, TenantContext.getTenantId(), message);
+                });
+
+        downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.GENERATING);
+
+        return new DownloadResponse(storageId);
+    }
+
+
+    @SneakyThrows
+    public ImportResponse importDataset(String userId, byte[] archive) {
+
+        DatasetExchangeArchiveReader datasetExchangeArchiveReader = new DatasetExchangeArchiveReader(userId);
+        try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
+            zipEntryIterator.forEachRemaining(datasetExchangeArchiveReader::handleZipEntryData);
+        }
+        DatasetExchangeImportModel importModel = datasetExchangeArchiveReader.getDatasetExchangeImportModel();
+
+        TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importModel.getImportTemplateResult());
+        for (DatasetExchangeImportModel.Dossier dossier : importModel.getDossiers()) {
+
+            String dossierId = getDossierForImport(templateImportInfo, dossier, userId);
+            for (DatasetExchangeImportModel.File file : dossier.getFiles()) {
+                uploadService.processSingleFile(dossierId, file.getName(), file.getData(), false, false);
+            }
+        }
+        return new ImportResponse(templateImportInfo.getDossierTemplateId());
+    }
+
+
+    private String getDossierForImport(TemplateImportInfo templateImportInfo, DatasetExchangeImportModel.Dossier dossier, String userId) {
+
+        String dossierTemplateId = templateImportInfo.getDossierTemplateId();
+
+        CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
+                .dossierName(dossier.getName())
+                .description(dossier.getName())
+                .downloadFileTypes(Collections.emptySet())
+                .dossierTemplateId(dossierTemplateId)
+                .requestingUser(userId)
+                .reportTemplateIds(reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
+                                           .stream()
+                                           .map(ReportTemplateEntity::getTemplateId)
+                                           .toList())
+                .build();
+
+        Dossier importedDossier = null;
+        int retries = 0;
+        while (importedDossier == null && retries < 100) {
+            try {
+                importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
+            } catch (ConflictException e) {
+                retries++;
+                request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
+            }
+        }
+        if (importedDossier == null) {
+            throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
+        }
+        return importedDossier.getId();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierExportService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
 
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
@@ -9,6 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
@@ -33,29 +35,30 @@ public class DossierExportService {
     FileStatusManagementService fileStatusManagementService;
     EntityTypeExportService entityTypeExportService;
     ObjectMapper mapper;
+    FileStatusService fileStatusService;
 
 
     @SneakyThrows
     @Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
     public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
 
-        List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
+        List<String> fileIdsInDossier = fileStatusManagementService.getDossierStatusIds(dossier.getId(), false)
+                .stream()
+                .filter(fileId -> request.fileIds().isEmpty() || request.fileIds().contains(fileId))
+                .toList();
 
-        if (!request.dossierIds().contains(dossier.getId()) //
-            && files.stream()
-                    .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
-            // dossier has no files in requested files and dossier not explicitly requested -> don't export it.
+        if (fileIdsInDossier.isEmpty()) {
             return;
         }
 
         Path dossierFolder = folder.resolve(dossier.getId());
         archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
 
-        for (FileModel fileEntity : files) {
-            if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
-                continue;
-            }
-            fileExportService.addFileToArchive(archiver, dossierFolder, request, fileEntity);
+        for (String fileId : fileIdsInDossier) {
+
+            FileModel file = fileStatusService.getStatus(fileId);
+
+            fileExportService.addFileToArchive(archiver, dossierFolder, request, file);
         }
 
         List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java

@@ -67,7 +67,7 @@ public class DossierImportService {
 
         Dossier importedDossier = null;
         int retries = 0;
-        while (importedDossier == null && retries < 10) {
+        while (importedDossier == null && retries < 100) {
             try {
                 importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
             } catch (ConflictException e) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java

@@ -9,6 +9,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
+import java.util.Optional;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
@@ -17,7 +18,9 @@ import org.springframework.util.FileSystemUtils;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DateFormatsExportModel;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DateFormatsEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity;
@@ -40,6 +43,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Compone
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ComponentDefinitionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
@@ -49,6 +53,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.DossierAttributeConfigMapper;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileAttributeConfigMapper;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -63,10 +69,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -87,6 +95,7 @@ public class DossierTemplateExportService {
     FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
     LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
     RulesPersistenceService rulesPersistenceService;
+    DateFormatsPersistenceService dateFormatsPersistenceService;
     FileManagementStorageService fileManagementStorageService;
     ReportTemplatePersistenceService reportTemplatePersistenceService;
     ComponentMappingService componentMappingService;
@@ -117,12 +126,13 @@ public class DossierTemplateExportService {
 
     private void addToExportDownloadQueue(ExportDownloadMessage downloadJob, int priority) {
 
-        rabbitTemplate.convertAndSend(MessagingConfiguration.EXPORT_DOWNLOAD_QUEUE, downloadJob, message -> {
+        rabbitTemplate.convertAndSend(MessagingConfiguration.EXPORT_DOWNLOAD_REQUEST_EXCHANGE, TenantContext.getTenantId(), downloadJob, message -> {
             message.getMessageProperties().setPriority(priority);
             return message;
         });
     }
 
+
     @Observed(name = "DossierTemplateExportService", contextualName = "export-dossier-template")
     public long createDownloadArchive(ExportDownloadMessage downloadJob) throws IOException {
 
@@ -142,8 +152,10 @@ public class DossierTemplateExportService {
 
     }
 
+
+    @SneakyThrows
     @Observed(name = "DossierTemplateExportService", contextualName = "write-dossier-template-to-archive")
-    public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) throws IOException {
+    public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) {
 
         // add dossier template name and meta data
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
@@ -173,13 +185,16 @@ public class DossierTemplateExportService {
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
                                                                                       getFilename(ExportFilename.DOSSIER_ATTRIBUTES_CONFIG, JSON_EXT),
                                                                                       objectMapper.writeValueAsBytes(convert(dossierAttributesConfig,
-                                                                                                                             DossierAttributeConfig.class))));
+                                                                                                                             DossierAttributeConfig.class,
+                                                                                                                             new DossierAttributeConfigMapper()))));
 
         // add file attribute configs
         List<FileAttributeConfigEntity> fileAttributeConfigList = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplate.getId());
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
                                                                                       getFilename(ExportFilename.FILE_ATTRIBUTE_CONFIG, JSON_EXT),
-                                                                                      objectMapper.writeValueAsBytes(convert(fileAttributeConfigList, FileAttributeConfig.class))));
+                                                                                      objectMapper.writeValueAsBytes(convert(fileAttributeConfigList,
+                                                                                                                             FileAttributeConfig.class,
+                                                                                                                             new FileAttributeConfigMapper()))));
 
         // add legal basis mapping
         List<LegalBasisEntity> legalBasisMappingList = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplate.getId());
@@ -199,7 +214,11 @@ public class DossierTemplateExportService {
         }
 
         // add rule set
-        RuleSetEntity ruleSet = rulesPersistenceService.getRules(dossierTemplate.getId(), RuleFileType.ENTITY);
+        Optional<RuleSetEntity> ruleSetOptional = rulesPersistenceService.getRules(dossierTemplate.getId(), RuleFileType.ENTITY);
+        if (ruleSetOptional.isEmpty()) {
+            throw new NotFoundException(String.format("No rule file of type %s found for dossierTemplateId %s", RuleFileType.ENTITY, dossierTemplate.getId()));
+        }
+        RuleSetEntity ruleSet = ruleSetOptional.get();
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
                                                                                       getFilename(ExportFilename.RULES, RULE_EXT),
                                                                                       ruleSet.getValue().getBytes(StandardCharsets.UTF_8)));
@@ -207,9 +226,14 @@ public class DossierTemplateExportService {
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
                                                                                       getFilename(ExportFilename.RULES, JSON_EXT),
                                                                                       objectMapper.writeValueAsBytes(rulesExportModel)));
+
         // add component rule set
         try {
-            RuleSetEntity componentRuleSet = rulesPersistenceService.getRules(dossierTemplate.getId(), RuleFileType.COMPONENT);
+            Optional<RuleSetEntity> componentRuleSetOptional = rulesPersistenceService.getRules(dossierTemplate.getId(), RuleFileType.COMPONENT);
+            if (componentRuleSetOptional.isEmpty()) {
+                throw new NotFoundException(String.format("No rule file of type %s found for dossierTemplateId %s", RuleFileType.COMPONENT, dossierTemplate.getId()));
+            }
+            RuleSetEntity componentRuleSet = componentRuleSetOptional.get();
             fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
                                                                                           getFilename(ExportFilename.COMPONENT_RULES, RULE_EXT),
                                                                                           componentRuleSet.getValue().getBytes(StandardCharsets.UTF_8)));
@@ -221,6 +245,24 @@ public class DossierTemplateExportService {
             log.debug("No component rules present.");
         }
 
+        // add date formats
+        try {
+            Optional<DateFormatsEntity> dateFormatsOptional = dateFormatsPersistenceService.getDateFormats(dossierTemplate.getId());
+            if (dateFormatsOptional.isEmpty()) {
+                throw new NotFoundException(String.format("No date formats found for dossierTemplateId %s", dossierTemplate.getId()));
+            }
+            DateFormatsEntity dateFormats = dateFormatsOptional.get();
+            fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
+                                                                                          getFilename(ExportFilename.DATE_FORMATS, TXT_EXT),
+                                                                                          dateFormats.getValue().getBytes(StandardCharsets.UTF_8)));
+            DateFormatsExportModel dateFormatsExportModel = DateFormatsExportModel.builder().version(dateFormats.getVersion()).build();
+            fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
+                                                                                          getFilename(ExportFilename.DATE_FORMATS, JSON_EXT),
+                                                                                          objectMapper.writeValueAsBytes(dateFormatsExportModel)));
+        } catch (NotFoundException e) {
+            log.debug("No date formats present.");
+        }
+
         //N files with the related report templates
         List<ReportTemplateEntity> reportTemplateList = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplate.getId());
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
@@ -267,7 +309,7 @@ public class DossierTemplateExportService {
         // components
         List<ComponentDefinitionEntity> componentDefinitions = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierTemplate.getId());
         for (ComponentDefinitionEntity componentDefinition : componentDefinitions) {
-            fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(componentDefinition.getTechnicalName(),
+            fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(resolveFolder(folder, componentDefinition.getTechnicalName()),
                                                                                           getFilename(ExportFilename.COMPONENTS, JSON_EXT),
                                                                                           objectMapper.writeValueAsBytes(componentDefinition)));
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateImportService.java

@@ -21,6 +21,12 @@ import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ComponentMappingImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.EntityTypeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DossierTemplateArchiveReader;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
@@ -31,17 +37,17 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ComponentMappingImportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DossierTemplateArchiveReader;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
+import com.iqser.red.service.persistence.management.v1.processor.service.DateFormatsValidationService;
+import com.iqser.red.service.persistence.management.v1.processor.service.LayoutParsingTypeProvider;
+import com.iqser.red.service.persistence.management.v1.processor.service.DefaultDateFormatsProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReportTemplateService;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ComponentDefinitionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
@@ -56,12 +62,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkMo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DateFormatPatternErrorMessage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.DroolsValidation;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
@@ -83,6 +91,7 @@ public class DossierTemplateImportService {
     DossierTemplateRepository dossierTemplateRepository;
     LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
     RulesPersistenceService rulesPersistenceService;
+    DateFormatsPersistenceService dateFormatsPersistenceService;
     DossierTemplatePersistenceService dossierTemplatePersistenceService;
     DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
     FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
@@ -92,16 +101,26 @@ public class DossierTemplateImportService {
     ReportTemplateService reportTemplateService;
     ReportTemplatePersistenceService reportTemplatePersistenceService;
     RulesValidationService rulesValidationService;
+    DateFormatsValidationService dateFormatsValidationService;
     StorageService storageService;
     FileManagementServiceSettings settings;
     ComponentMappingService componentMappingService;
     ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
     EntityTypeImportService entityTypeImportService;
+    SystemManagedTypesImport systemManagedTypesImport;
+    LayoutParsingTypeProvider layoutParsingTypeProvider;
+    CurrentApplicationTypeProvider currentApplicationTypeProvider;
+    DefaultDateFormatsProvider defaultDateFormatsProvider;
 
 
     public String importDossierTemplate(ImportDossierTemplateRequest request) {
 
-        ImportTemplateResult archiveResult = readDossierTemplateImportArchive(request);
+        ImportTemplateResult archiveResult;
+        try {
+            archiveResult = readDossierTemplateImportArchive(request);
+        } catch (Exception e) {
+            throw new BadRequestException("The import process failed in step: 0 - while reading the archive content with error: " + e.getMessage());
+        }
 
         return importDossierTemplate(archiveResult).getDossierTemplateId();
     }
@@ -121,13 +140,19 @@ public class DossierTemplateImportService {
         return dossierTemplateArchiveReader.buildResult();
     }
 
+
     @Observed(name = "DossierTemplateImportService", contextualName = "import-template")
     public TemplateImportInfo importDossierTemplate(ImportTemplateResult request) {
 
         long start = System.currentTimeMillis();
         String dossierTemplateId;
+        int importStep = 1;
         var dossierTemplateMeta = request.getDossierTemplate();
 
+        if (dossierTemplateMeta.getLayoutParsingType() == null) {
+            dossierTemplateMeta.setLayoutParsingType(layoutParsingTypeProvider.deferFromCurrentApplicationType());
+        }
+
         TemplateImportInfo templateImportInfo = TemplateImportInfo.builder().build();
 
         DossierTemplateEntity existingDossierTemplate = null;
@@ -136,217 +161,268 @@ public class DossierTemplateImportService {
             existingDossierTemplate = dossierTemplateOptional.orElse(null);
         }
 
-        if (existingDossierTemplate != null) {
-            dossierTemplateId = existingDossierTemplate.getId();
+        try {
 
-            // override the existing dossier template
-            updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId());
-            dossierTemplateRepository.save(existingDossierTemplate);
+            if (existingDossierTemplate != null) {
+                dossierTemplateId = existingDossierTemplate.getId();
 
-            existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(existingDossierTemplate)
-                                                                                                   .name()));
+                // override the existing dossier template
+                updateDossierTemplateMeta(existingDossierTemplate, dossierTemplateMeta, request.getUserId());
+                dossierTemplateRepository.save(existingDossierTemplate);
 
-            // set colors
-            this.setColors(dossierTemplateId, request.getColors());
+                existingDossierTemplate.setDossierTemplateStatus(DossierTemplateStatus.valueOf(dossierTemplatePersistenceService.computeDossierTemplateStatus(
+                        existingDossierTemplate).name()));
 
-            // set watermarks
-            if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
-                Set<String> toSetWatermarks = request.getWatermarks()
-                        .stream()
-                        .map(WatermarkModel::getName)
-                        .filter(Objects::nonNull)
-                        .collect(Collectors.toSet());
-                var currentWatermarkConfigs = watermarkService.getWatermarksForDossierTemplateId(dossierTemplateId);
-                Set<WatermarkEntity> configsToRemove = currentWatermarkConfigs.stream()
-                        .filter(c -> !toSetWatermarks.contains(c.getName()))
-                        .collect(Collectors.toSet());
-                var watermarkNameToEntity = currentWatermarkConfigs.stream()
-                        .collect(Collectors.toMap(WatermarkEntity::getName, Function.identity()));
+                // set colors
+                importStep = 2;
+                this.setColors(dossierTemplateId, request.getColors());
 
-                request.getWatermarks()
-                        .forEach(watermark -> {
-                            log.info("watermark to add: " + watermark.getName());
-                            Long initialId = watermark.getId();
-                            if (!watermarkNameToEntity.isEmpty() && watermarkNameToEntity.get(watermark.getName()) != null) {
+                // set watermarks
+                if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
+                    importStep = 3;
+                    Set<String> toSetWatermarks = request.getWatermarks()
+                            .stream()
+                            .map(WatermarkModel::getName)
+                            .filter(Objects::nonNull)
+                            .collect(Collectors.toSet());
+                    var currentWatermarkConfigs = watermarkService.getWatermarksForDossierTemplateId(dossierTemplateId);
+                    Set<WatermarkEntity> configsToRemove = currentWatermarkConfigs.stream()
+                            .filter(c -> !toSetWatermarks.contains(c.getName()))
+                            .collect(Collectors.toSet());
+                    var watermarkNameToEntity = currentWatermarkConfigs.stream()
+                            .collect(Collectors.toMap(WatermarkEntity::getName, Function.identity()));
 
-                                watermark.setId(watermarkNameToEntity.get(watermark.getName()).getId());
-                            } else {
+                    request.getWatermarks()
+                            .forEach(watermark -> {
+                                log.info("watermark to add: " + watermark.getName());
+                                Long initialId = watermark.getId();
+                                if (!watermarkNameToEntity.isEmpty() && watermarkNameToEntity.get(watermark.getName()) != null) {
+
+                                    watermark.setId(watermarkNameToEntity.get(watermark.getName()).getId());
+                                } else {
+                                    watermark.setId(null);
+                                }
+                                watermark.setDossierTemplateId(dossierTemplateId);
+                                var entity = watermarkService.createOrUpdateWatermark(watermark);
+                                templateImportInfo.getLongMapping().put(initialId, entity.getId());
+                            });
+                    configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId()));
+                }
+
+                // dossier status
+                importStep = 4;
+                if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
+                    this.updateDossierStates(request, dossierTemplateId);
+                } else { // no states to add, delete current states
+                    List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
+                    currentStates.forEach(state -> dossierStatusPersistenceService.deleteDossierStatus(state.getId(), null));
+                }
+
+                // update dossier attributes
+                importStep = 5;
+                if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
+                    this.updateDossierAttributes(request, dossierTemplateId, templateImportInfo);
+                } else { // no dossier attributes to add, but delete existing ones
+                    var currentConfigs = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
+                    currentConfigs.forEach(da -> dossierAttributeConfigPersistenceService.deleteDossierAttribute(da.getId()));
+                }
+
+                //update file attributes
+                importStep = 6;
+                if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
+                    this.updateFileAttributes(request, dossierTemplateId, templateImportInfo);
+                } else { // no file attributes to add, but delete existing
+                    var currentConfigs = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
+                    currentConfigs.forEach(fa -> fileAttributeConfigPersistenceService.deleteFileAttribute(fa.getId()));
+                }
+
+                //set report templates
+                importStep = 7;
+                var existingReports = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplateId);
+
+                List<String> reportsUpdated = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
+                    request.getReportTemplateUploadRequests()
+                            .forEach(reportRequest -> {
+                                reportRequest.setDossierTemplateId(dossierTemplateId);
+                                var report = reportTemplateService.uploadTemplate(reportRequest);
+                                reportsUpdated.add(report.getTemplateId());
+                            });
+                }
+
+                // delete the reports that were not in the import
+                existingReports.forEach(r -> {
+                    String storageId = r.getStorageId();
+                    if (!reportsUpdated.contains(r.getTemplateId())) {
+                        storageService.deleteObject(TenantContext.getTenantId(), storageId);
+                        reportTemplatePersistenceService.delete(r.getTemplateId());
+                    }
+                });
+
+                // set legal basis
+                importStep = 8;
+                if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
+                    legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
+                } else { // delete existing
+                    legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId);
+                }
+
+                // file attribute general configuration
+                if (request.getFileAttributesGeneralConfiguration() != null) {
+                    importStep = 9;
+                    fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
+                                                                                         convert(request.getFileAttributesGeneralConfiguration(),
+                                                                                                 FileAttributesGeneralConfigurationEntity.class));
+                }
+
+                // component definitions
+                importStep = 10;
+                if (CollectionUtils.isNotEmpty(request.getComponentDefinitions())) {
+                    this.updateComponents(request, dossierTemplateId);
+                } else { // no components to add, but remove existing ones
+                    List<ComponentDefinitionEntity> currentComponents = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
+                    this.deleteComponents(currentComponents);
+                }
+
+                //set component mappings
+                importStep = 11;
+                setComponentMappings(dossierTemplateId, request.getComponentMappings());
+
+                importStep = 12;
+                entityTypeImportService.updateTypes(dossierTemplateId,
+                                                    null,
+                                                    entityTypeImportService.updateAndCleanSystemManagedFromImportTypes(request.getEntityTypeImportModel(), dossierTemplateId));
+
+            } else {
+
+                // creates new dossier template
+                if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
+                    throw new ConflictException("DossierTemplate name must be set");
+                }
+                dossierTemplateMeta.setId("");
+                this.validateDossierTemplateName(dossierTemplateMeta);
+                DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
+
+                // order is important
+                BeanUtils.copyProperties(dossierTemplateMeta, dossierTemplateEntity);
+                dossierTemplateEntity.setId(UUID.randomUUID().toString());
+                dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+                dossierTemplateEntity.setCreatedBy(request.getUserId());
+
+                var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity);
+                loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate));
+
+                dossierTemplateId = loadedDossierTemplate.getId();
+
+                // set colors
+                importStep = 2;
+                this.setColors(dossierTemplateId, request.getColors());
+
+                // set watermarks
+                if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
+                    importStep = 3;
+                    request.getWatermarks()
+                            .forEach(watermark -> {
+                                Long initialId = watermark.getId();
+                                watermark.setDossierTemplateId(dossierTemplateId);
                                 watermark.setId(null);
-                            }
-                            watermark.setDossierTemplateId(dossierTemplateId);
-                            var entity = watermarkService.createOrUpdateWatermark(watermark);
-                            templateImportInfo.getLongMapping().put(initialId, entity.getId());
-                        });
-                configsToRemove.forEach(watermark -> watermarkService.deleteWatermark(watermark.getId()));
-            }
-
-            // dossier status
-            if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
-                this.updateDossierStates(request, dossierTemplateId);
-            } else { // no states to add, delete current states
-                List<DossierStatusInfo> currentStates = dossierStatusPersistenceService.getAllDossierStatusForTemplate(dossierTemplateId);
-                currentStates.forEach(state -> dossierStatusPersistenceService.deleteDossierStatus(state.getId(), null));
-            }
-
-            // update dossier attributes
-            if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
-                this.updateDossierAttributes(request, dossierTemplateId, templateImportInfo);
-            } else { // no dossier attributes to add, but delete existing ones
-                var currentConfigs = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
-                currentConfigs.forEach(da -> dossierAttributeConfigPersistenceService.deleteDossierAttribute(da.getId()));
-            }
-
-            //update file attributes
-            if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
-                this.updateFileAttributes(request, dossierTemplateId, templateImportInfo);
-            } else { // no file attributes to add, but delete existing
-                var currentConfigs = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
-                currentConfigs.forEach(fa -> fileAttributeConfigPersistenceService.deleteFileAttribute(fa.getId()));
-            }
-
-            entityTypeImportService.updateTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
-
-            if (CollectionUtils.isNotEmpty(request.getComponentDefinitions())) {
-                this.updateComponents(request, dossierTemplateId);
-            } else { // no components to add, but remove existing ones
-                List<ComponentDefinitionEntity> currentComponents = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
-                this.deleteComponents(currentComponents);
-            }
-
-            //set report templates
-            var existingReports = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplateId);
-
-            List<String> reportsUpdated = new ArrayList<>();
-            if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
-                request.getReportTemplateUploadRequests()
-                        .forEach(reportRequest -> {
-                            reportRequest.setDossierTemplateId(dossierTemplateId);
-                            var report = reportTemplateService.uploadTemplate(reportRequest);
-                            reportsUpdated.add(report.getTemplateId());
-                        });
-            }
-
-            // delete the reports that were not in the import
-            existingReports.forEach(r -> {
-                String storageId = r.getStorageId();
-                if (!reportsUpdated.contains(r.getTemplateId())) {
-                    storageService.deleteObject(TenantContext.getTenantId(), storageId);
-                    reportTemplatePersistenceService.delete(r.getTemplateId());
+                                var en = watermarkService.createOrUpdateWatermark(watermark);
+                                templateImportInfo.getLongMapping().put(initialId, en.getId());
+                            });
                 }
-            });
 
-            // set legal basis
-            if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
-                legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
-            } else { // delete existing
-                legalBasisMappingPersistenceService.deleteLegalBasis(dossierTemplateId);
-            }
-
-        } else {
-            // creates new dossier template
-            if (StringUtils.isEmpty(dossierTemplateMeta.getName())) {
-                throw new ConflictException("DossierTemplate name must be set");
-            }
-            dossierTemplateMeta.setId("");
-            this.validateDossierTemplateName(dossierTemplateMeta);
-            DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
-
-            // order is important
-            BeanUtils.copyProperties(dossierTemplateMeta, dossierTemplateEntity);
-            dossierTemplateEntity.setId(UUID.randomUUID().toString());
-            dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-            dossierTemplateEntity.setCreatedBy(request.getUserId());
-            //set rules
-
-            var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity);
-            loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate));
-
-            dossierTemplateId = loadedDossierTemplate.getId();
-            // set colors
-            this.setColors(dossierTemplateId, request.getColors());
-
-            // set watermarks
-            if (CollectionUtils.isNotEmpty(request.getWatermarks())) {
-                request.getWatermarks()
-                        .forEach(watermark -> {
-                            Long initialId = watermark.getId();
-                            watermark.setDossierTemplateId(dossierTemplateId);
-                            watermark.setId(null);
-                            var en = watermarkService.createOrUpdateWatermark(watermark);
-                            templateImportInfo.getLongMapping().put(initialId, en.getId());
-                        });
-            }
-
-            // dossier status
-            if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
-                request.getDossierStatusInfos()
-                        .forEach(state -> {
-                            state.setId(null);
-                            this.updateDossierStatus(dossierTemplateId, state);
-                        });
-            }
-
-            //set dossier attributes
-            if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
-                request.getDossierAttributesConfigs()
-                        .forEach(da -> {
-                            String initialId = da.getId();
-                            da.setId(null); // in order to create a new dossier attribute
-                            var e = dossierAttributeConfigPersistenceService.addOrUpdateDossierAttribute(dossierTemplateId, convert(da, DossierAttributeConfigEntity.class));
-                            templateImportInfo.getIdMapping().put(initialId, e.getId());
-                        });
-            }
-
-            //set file attributes
-            if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
-                request.getFileAttributesConfigs()
-                        .forEach(fa -> {
-                            String initialId = fa.getId();
-                            fa.setId(null); // in order to force creation of new file attribute
-                            var e = fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId, convert(fa, FileAttributeConfigEntity.class));
-                            templateImportInfo.getIdMapping().put(initialId, e.getId());
-                        });
-            }
-
-            //set types
-            entityTypeImportService.importEntityTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
-
-            // set components
-            if (CollectionUtils.isNotEmpty(request.getComponentDefinitions())) {
-                for (ComponentDefinition componentDefinition : request.getComponentDefinitions()) {
-                    ComponentDefinitionAddRequest componentDefinitionAddRequest = ComponentDefinitionAddRequest.builder()
-                            .displayName(componentDefinition.getDisplayName())
-                            .description(componentDefinition.getDescription())
-                            .technicalName(componentDefinition.getTechnicalName())
-                            .build();
-                    componentDefinitionPersistenceService.insert(componentDefinitionAddRequest, componentDefinition.getRank(), dossierTemplateId);
+                // dossier status
+                if (CollectionUtils.isNotEmpty(request.getDossierStatusInfos())) {
+                    importStep = 4;
+                    request.getDossierStatusInfos()
+                            .forEach(state -> {
+                                state.setId(null);
+                                this.updateDossierStatus(dossierTemplateId, state);
+                            });
                 }
+
+                //set dossier attributes
+                if (CollectionUtils.isNotEmpty(request.getDossierAttributesConfigs())) {
+                    importStep = 5;
+                    request.getDossierAttributesConfigs()
+                            .forEach(da -> {
+                                String initialId = da.getId();
+                                da.setId(null); // in order to create a new dossier attribute
+                                var e = dossierAttributeConfigPersistenceService.addOrUpdateDossierAttribute(dossierTemplateId, convert(da, DossierAttributeConfigEntity.class));
+                                templateImportInfo.getIdMapping().put(initialId, e.getId());
+                            });
+                }
+
+                //set file attributes
+                if (CollectionUtils.isNotEmpty(request.getFileAttributesConfigs())) {
+                    importStep = 6;
+                    request.getFileAttributesConfigs()
+                            .forEach(fa -> {
+                                String initialId = fa.getId();
+                                fa.setId(null); // in order to force creation of new file attribute
+                                var e = fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId, convert(fa, FileAttributeConfigEntity.class));
+                                templateImportInfo.getIdMapping().put(initialId, e.getId());
+                            });
+                }
+
+                //set report templates
+                if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
+                    importStep = 7;
+                    request.getReportTemplateUploadRequests()
+                            .forEach(reportRequest -> {
+                                reportRequest.setDossierTemplateId(dossierTemplateId);
+                                reportTemplateService.uploadTemplate(reportRequest);
+                            });
+                }
+                // set legal basis
+                if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
+                    importStep = 8;
+                    legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
+                }
+
+                // file attribute general configuration
+                if (request.getFileAttributesGeneralConfiguration() != null) {
+                    importStep = 9;
+                    fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
+                                                                                         convert(request.getFileAttributesGeneralConfiguration(),
+                                                                                                 FileAttributesGeneralConfigurationEntity.class));
+                }
+
+                // set components
+                if (CollectionUtils.isNotEmpty(request.getComponentDefinitions())) {
+                    importStep = 10;
+                    for (ComponentDefinition componentDefinition : request.getComponentDefinitions()) {
+                        ComponentDefinitionAddRequest componentDefinitionAddRequest = ComponentDefinitionAddRequest.builder()
+                                .displayName(componentDefinition.getDisplayName())
+                                .description(componentDefinition.getDescription())
+                                .technicalName(componentDefinition.getTechnicalName())
+                                .build();
+                        componentDefinitionPersistenceService.insert(componentDefinitionAddRequest, componentDefinition.getRank(), dossierTemplateId);
+                    }
+                }
+
+                //set component mappings
+                importStep = 11;
+                setComponentMappings(dossierTemplateId, request.getComponentMappings());
+
+                //set types
+                importStep = 12;
+                entityTypeImportService.addMissingSystemManagedTypesToImport(request.getEntityTypeImportModel(), systemManagedTypesImport.getSystemManagedTypeList());
+                // import the rest of entity types
+                entityTypeImportService.importEntityTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
+
             }
 
-            //set report templates
-            if (CollectionUtils.isNotEmpty(request.getReportTemplateUploadRequests())) {
-                request.getReportTemplateUploadRequests()
-                        .forEach(reportRequest -> {
-                            reportRequest.setDossierTemplateId(dossierTemplateId);
-                            reportTemplateService.uploadTemplate(reportRequest);
-                        });
-            }
-            // set legal basis
-            if (CollectionUtils.isNotEmpty(request.getLegalBases())) {
-                legalBasisMappingPersistenceService.setLegalBasisMapping(dossierTemplateId, request.getLegalBases());
-            }
+            importStep = 13;
+            setRulesWhenCompiled(request, dossierTemplateId);
+
+            importStep = 14;
+            setDataFormats(request, dossierTemplateId);
+
+        } catch (Exception e) {
+            throw new BadRequestException("The import process failed in step: " + importStep + " with error: " + e.getMessage());
         }
 
-        // file attribute general configuration
-        if (request.getFileAttributesGeneralConfiguration() != null) {
-            fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
-                                                                                 convert(request.getFileAttributesGeneralConfiguration(),
-                                                                                         FileAttributesGeneralConfigurationEntity.class));
-        }
-
-        setRulesWhenCompiled(request, dossierTemplateId);
-        setComponentMappings(dossierTemplateId, request.getComponentMappings());
-
         templateImportInfo.setDossierTemplateId(dossierTemplateId);
 
         long elapsedTime = System.currentTimeMillis() - start;
@@ -372,7 +448,8 @@ public class DossierTemplateImportService {
                                                                                      componentMapping.metadata().getFileName(),
                                                                                      componentMapping.metadata().getDelimiter(),
                                                                                      componentMapping.metadata().getEncoding(),
-                                                                                     tmpFile);
+                                                                                     tmpFile,
+                                                                                     componentMapping.metadata().getQuoteChar());
 
             componentMappingService.setVersion(createdMapping.getId(), componentMapping.metadata().getVersion());
             assert tmpFile.delete();
@@ -415,6 +492,23 @@ public class DossierTemplateImportService {
     }
 
 
+    private void setDataFormats(ImportTemplateResult request, String dossierTemplateId) {
+
+        String dateFormats = request.getDateFormats();
+        if (dateFormats == null && currentApplicationTypeProvider.isDocuMine()) {
+            dateFormats = defaultDateFormatsProvider.getDateFormats();
+        }
+        if (dateFormats != null) {
+            List<DateFormatPatternErrorMessage> dateFormatPatternErrorMessages = dateFormatsValidationService.validateDateFormats(dateFormats);
+            if (!dateFormatPatternErrorMessages.isEmpty()) {
+                throw new BadRequestException("The date formats file contains errors");
+            }
+            dateFormatsPersistenceService.setDateFormats(dateFormats, dossierTemplateId, request.getDateFormatsExportModel().getVersion());
+        }
+
+    }
+
+
     private void updateDossierTemplateMeta(DossierTemplateEntity dossierTemplateEntity, DossierTemplate dossierTemplate, String userId) {
 
         dossierTemplate.setId(dossierTemplateEntity.getId());
@@ -428,6 +522,7 @@ public class DossierTemplateImportService {
         dossierTemplateEntity.setModifiedBy(userId);
         dossierTemplateEntity.setValidFrom(dossierTemplate.getValidFrom() == null ? dossierTemplateEntity.getValidFrom() : dossierTemplate.getValidFrom());
         dossierTemplateEntity.setValidTo(dossierTemplate.getValidTo() == null ? dossierTemplateEntity.getValidTo() : dossierTemplate.getValidTo());
+        dossierTemplateEntity.setLayoutParsingType(layoutParsingTypeProvider.deferFromCurrentApplicationType());
 
         dossierTemplateEntity.setDownloadFileTypes(dossierTemplate.getDownloadFileTypes() == null || dossierTemplate.getDownloadFileTypes()
                 .isEmpty() ? dossierTemplateEntity.getDownloadFileTypes() : dossierTemplate.getDownloadFileTypes());
@@ -512,7 +607,7 @@ public class DossierTemplateImportService {
         request.getFileAttributesConfigs()
                 .forEach(fa -> {
                     String initialId = fa.getId();
-                    //check by id or by dossierTemplate and label and set the corresponding Id if found
+                    //check by id or by dossierTemplate and label and set the corresponding id if found
                     fa.setId(fileAttributeConfigPersistenceService.getFileAttributeByIdOrName(dossierTemplateId, fa.getId(), fa.getLabel()));
                     FileAttributeConfigEntity fileAttributeAdded = fileAttributeConfigPersistenceService.addOrUpdateFileAttribute(dossierTemplateId,
                                                                                                                                   convert(fa, FileAttributeConfigEntity.class));
@@ -532,6 +627,9 @@ public class DossierTemplateImportService {
 
     private void updateComponents(ImportTemplateResult request, String dossierTemplateId) {
 
+        List<ComponentDefinitionEntity> existingComponentDefinitions = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
+        existingComponentDefinitions.forEach(componentDefinition -> componentDefinitionPersistenceService.delete(componentDefinition.getId()));
+
         request.getComponentDefinitions()
                 .forEach(componentDefinition -> {
                     componentDefinition.setDossierTemplateId(dossierTemplateId);
@@ -542,9 +640,7 @@ public class DossierTemplateImportService {
 
     private void deleteComponents(List<ComponentDefinitionEntity> componentDefinitionEntities) {
 
-        componentDefinitionEntities.forEach(componentDefinition -> {
-            componentDefinitionPersistenceService.delete(componentDefinition.getId());
-        });
+        componentDefinitionEntities.forEach(componentDefinition -> componentDefinitionPersistenceService.delete(componentDefinition.getId()));
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/EntityTypeImportService.java

@@ -47,6 +47,7 @@ public class EntityTypeImportService {
         List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getTypesForDossierTemplate(dossierTemplateId)
                 .stream()
                 .filter(t -> t.getDossierId() == null)
+                .filter(t -> !t.isSystemManaged())
                 .collect(Collectors.toList());
 
         Set<String> currentTypesId = currentTypes.stream()
@@ -124,10 +125,77 @@ public class EntityTypeImportService {
                             returnedType.getType(),
                             DictionaryEntryType.FALSE_RECOMMENDATION);
 
-            dictionaryPersistenceService.setVersion(returnedType.getTypeId(), type.getVersion());
+            dictionaryPersistenceService.setVersions(returnedType.getTypeId(), type.getVersion(), type.getAiCreationVersion());
         }
     }
 
+    public void addMissingSystemManagedTypesToImport(EntityTypeImportModel importModel, List<Type> systemManagedTypes) {
+        Set<String> defaultSystemManagedTypesByName = systemManagedTypes.stream()
+                .map(Type::getType)
+                .collect(Collectors.toSet());
+        importModel.getTypes()
+                .stream()
+                .filter(t -> defaultSystemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
+                .forEach(t -> {
+                    if (!t.isSystemManaged()) {
+                        t.setSystemManaged(true);
+                    }
+                });
+
+        Set<String> systemManagedTypesFromImport = importModel.getTypes()
+                .stream()
+                .filter(t -> defaultSystemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
+                .map(Type::getType)
+                .collect(Collectors.toSet());
+        List<Type> systemManagedTypesNotIncludedInTheImport = systemManagedTypes.stream()
+                .filter(type -> !systemManagedTypesFromImport.contains(type.getType()))
+                .collect(Collectors.toList());
+        importModel.getTypes().addAll(systemManagedTypesNotIncludedInTheImport);
+    }
+
+    public EntityTypeImportModel updateAndCleanSystemManagedFromImportTypes(EntityTypeImportModel importModel, String dossierTemplateId) {
+
+        List<TypeEntity> currentSystemManagedTypes = dossierTemplatePersistenceService.getTypesForDossierTemplate(dossierTemplateId)
+                .stream()
+                .filter(t -> t.getDossierId() == null)
+                .filter(t -> t.isSystemManaged())
+                .collect(Collectors.toList());
+
+        var currentSystemManagedTypesNameToTypeMap = currentSystemManagedTypes.stream()
+                .collect(Collectors.toMap(TypeEntity::getType, Function.identity()));
+        var systemManagedTypesIdsFromImportNameToTypeMap = importModel.getTypes()
+                .stream()
+                .filter(t -> currentSystemManagedTypesNameToTypeMap.keySet().contains(t.getType()) || t.isSystemManaged())
+                .collect(Collectors.toMap(Type::getType, Function.identity()));
+
+        if (!systemManagedTypesIdsFromImportNameToTypeMap.isEmpty()) {
+            systemManagedTypesIdsFromImportNameToTypeMap.entrySet().forEach(entry -> {
+                var typeId = currentSystemManagedTypesNameToTypeMap.get(entry.getKey()).getId();
+                entry.getValue().setDossierTemplateId(dossierTemplateId);
+                entry.getValue().setDossierId(null);
+                dictionaryManagementService.updateTypeValue(typeId, entry.getValue());
+
+                this.addEntries(importModel.getEntries(), importModel.getDeletedEntries(), typeId, entry.getKey(), DictionaryEntryType.ENTRY);
+                this.addEntries(importModel.getFalsePositives(), importModel.getDeletedFalsePositives(), typeId, entry.getKey(), DictionaryEntryType.FALSE_POSITIVE);
+                this.addEntries(importModel.getFalseRecommendations(), importModel.getDeletedFalseRecommendations(), typeId, entry.getKey(), DictionaryEntryType.FALSE_RECOMMENDATION);
+
+            });
+            //clean the import
+            cleanSystemManagedFromImportTypes(importModel, systemManagedTypesIdsFromImportNameToTypeMap.keySet());
+        }
+        return importModel;
+    }
+
+    private void cleanSystemManagedFromImportTypes(EntityTypeImportModel importModel, Set<String> systemManagedTypesIdsFromImport) {
+        importModel.getTypes().removeIf(t -> systemManagedTypesIdsFromImport.contains(t.getType()));
+
+        importModel.getEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+    }
 
     private void enrichObservation(EntityTypeImportModel importModel) {
 
@@ -145,28 +213,10 @@ public class EntityTypeImportService {
 
     private void deleteTypes(List<TypeEntity> currentTypes, Set<String> typeIdsAdded) {
 
-        Set<String> currentTypesIdSystemManaged = currentTypes.stream()
-                .filter(TypeEntity::isSystemManaged)
-                .map(TypeEntity::getId)
-                .collect(Collectors.toSet());
-
-        // for types system managed just delete the entries
-        Set<String> entriesToRemove = currentTypesIdSystemManaged.stream()
-                .filter(t -> !typeIdsAdded.contains(t))
-                .collect(Collectors.toSet());
-        entriesToRemove.forEach(typeId -> {
-            var currentVersion = dictionaryManagementService.getCurrentVersion(typeId);
-            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
-            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
-            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
-            dictionaryPersistenceService.incrementVersion(typeId);
-            typeIdsAdded.add(typeId); // added to the list, since the type can not be deleted
-        });
         Set<String> typesToRemove = currentTypes.stream()
                 .filter(t -> !t.isDeleted()) // remove the ones already soft deleted
                 .map(TypeEntity::getId)
                 .filter(t -> !typeIdsAdded.contains(t)) // exclude the type ids already added from the import
-                .filter(t -> !currentTypesIdSystemManaged.contains(t)) // exclude the types system managed
                 .collect(Collectors.toSet());
         typesToRemove.forEach(dictionaryManagementService::deleteType);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java

@@ -3,11 +3,14 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
@@ -22,6 +25,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadRes
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
@@ -74,7 +78,7 @@ public class FileExchangeExportService {
 
     private void addToExportDownloadQueue(ExportDownloadMessage downloadJob) {
 
-        rabbitTemplate.convertAndSend(MessagingConfiguration.EXPORT_DOWNLOAD_QUEUE, downloadJob, message -> {
+        rabbitTemplate.convertAndSend(MessagingConfiguration.EXPORT_DOWNLOAD_REQUEST_EXCHANGE, TenantContext.getTenantId(), downloadJob, message -> {
             message.getMessageProperties().setPriority(1);
             return message;
         });
@@ -91,13 +95,18 @@ public class FileExchangeExportService {
 
             dossierTemplateExportService.addDossierTemplateToArchive(archiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate);
 
-            for (Dossier dossierEntity : dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplate.getId(), true, false)) {
+            Iterable<String> dossierIds;
+            if (request.dossierIds().isEmpty()) {
+                dossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplate.getId(), true, false);
+            } else {
+                dossierIds = request.dossierIds();
+            }
 
-                if (!request.dossierIds().isEmpty() && !request.dossierIds().contains(dossierEntity.getId())) {
-                    continue;
-                }
+            for (String dossierId : dossierIds) {
 
-                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
+                Dossier dossier = dossierManagementService.getDossierById(dossierId, true, false);
+
+                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossier);
             }
 
             storeZipFile(downloadJob.getStorageId(), archiver);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExportService.java

@@ -29,7 +29,6 @@ import lombok.extern.slf4j.Slf4j;
 public class FileExportService {
 
     ManualChangesExportService manualChangesExportService;
-    ComponentLogService componentLogService;
     ObjectMapper mapper;
     FileManagementStorageService storageService;
 
@@ -41,16 +40,15 @@ public class FileExportService {
         Path fileFolder = folder.resolve(file.getId());
 
         if (!request.excludeAnalysisLogs()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
-                                                                        mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
-            if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
+            if (storageService.entityLogExists(file.getDossierId(), file.getId())) {
+                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                            buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
+                                                                            mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
+            }
+            if (storageService.componentLogExists(file.getDossierId(), file.getId())) {
                 archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
                                                                             buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
                                                                             mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));
-                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                            buildFileName(file, FileExchangeNames.COMPONENT_OVERRIDES),
-                                                                            mapper.writeValueAsBytes(componentLogService.getOverrides(file.getDossierId(), file.getId()))));
             }
         }
 
@@ -62,7 +60,10 @@ public class FileExportService {
             addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
             addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
             addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.LLM_NER_ENTITIES);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.AZURE_NER_ENTITIES);
             addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
+            addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.CHUNKS);
         }
 
         addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileImportService.java

@@ -73,7 +73,7 @@ public class FileImportService {
         }
         if (file.getOverrides() != null) {
             file.getOverrides()
-                    .forEach(componentOverride -> componentLogService.addOverride(dossierId, fileId, componentOverride));
+                    .forEach(componentOverride -> componentLogService.overrideComponent(dossierId, fileId, componentOverride));
         }
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/SystemManagedTypesImport.java

@@ -0,0 +1,50 @@
+package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class SystemManagedTypesImport {
+
+    private final ObjectMapper objectMapper = ObjectMapperFactory.create();
+    private List<Type> rmSystemManagedTypeList = new ArrayList<>();
+
+
+    @PostConstruct
+    public void init() {
+
+        Resource redactManagerResource = new ClassPathResource("files/systemtypes/rm/systemTypes.json");
+
+        try {
+            rmSystemManagedTypeList = objectMapper.readValue(redactManagerResource.getContentAsByteArray(), new TypeReference<>() {
+            });
+
+        } catch (IOException e) {
+            log.info("Could not read the system managed types for  dossier templates error: {}", e.getMessage());
+        }
+    }
+
+
+    public List<Type> getSystemManagedTypeList() {
+
+        return this.rmSystemManagedTypeList;
+    }
+
+}