Merge branch 'RED-10691' into 'release/2.589.x'

RED-10691: 500 when approving file in a dossier where dossier owner lost manager role

See merge request redactmanager/persistence-service!929

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ApplicationConfigurationController.java

@@ -9,11 +9,17 @@ import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.ApplicationConfigurationResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
+import jakarta.persistence.Column;
 import jakarta.validation.Valid;
+import lombok.Builder;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -23,14 +29,24 @@ import lombok.extern.slf4j.Slf4j;
 public class ApplicationConfigurationController implements ApplicationConfigurationResource {
 
     private final ApplicationConfigService applicationConfigService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
     @PreAuthorize("hasAuthority('" + WRITE_APP_CONFIG + "')")
     public ApplicationConfig createOrUpdateAppConfig(@Valid @RequestBody ApplicationConfig appConfig) {
 
-        return MagicConverter.convert(applicationConfigService.saveApplicationConfiguration(MagicConverter.convert(appConfig, ApplicationConfigurationEntity.class)),
+        var result = MagicConverter.convert(applicationConfigService.saveApplicationConfiguration(convert(appConfig)),
                                       ApplicationConfig.class);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId("ApplicationConfig")
+                                              .category(AuditCategory.SETTINGS.name())
+                                              .message("Application config has been changed.")
+                                              .build());
+
+        return result;
     }
 
 
@@ -41,4 +57,18 @@ public class ApplicationConfigurationController implements ApplicationConfigurat
         return MagicConverter.convert(applicationConfigService.getApplicationConfig(), ApplicationConfig.class);
     }
 
+
+    private ApplicationConfigurationEntity convert(ApplicationConfig appConfig){
+        var entity = ApplicationConfigurationEntity.builder()
+                .downloadCleanupDownloadFilesHours(appConfig.getDownloadCleanupDownloadFilesHours())
+                .downloadCleanupNotDownloadFilesHours(appConfig.getDownloadCleanupNotDownloadFilesHours())
+                .softDeleteCleanupTime(appConfig.getSoftDeleteCleanupTime())
+                .build();
+
+        if(appConfig.getHardDeleteCleanupRetryTime() != null){
+            entity.setHardDeleteCleanupRetryTime(appConfig.getHardDeleteCleanupRetryTime());
+        }
+        return entity;
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/CustomPermissionMappingController.java

@@ -11,8 +11,12 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.acl.custom.service.CustomPermissionService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.CustomPermissionMappingResource;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.permission.CustomPermissionMappingModel;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import lombok.RequiredArgsConstructor;
 
@@ -21,6 +25,7 @@ import lombok.RequiredArgsConstructor;
 public class CustomPermissionMappingController implements CustomPermissionMappingResource {
 
     private final CustomPermissionService customPermissionService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
@@ -36,6 +41,14 @@ public class CustomPermissionMappingController implements CustomPermissionMappin
     public void saveCustomPermissionMappings(@PathVariable(TARGET_OBJECT_NAME) String targetObject, @RequestBody List<CustomPermissionMappingModel> customPermissionMappingModels) {
 
         customPermissionService.saveCustomPermissionMappings(targetObject, customPermissionMappingModels);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(targetObject)
+                                              .category(AuditCategory.SETTINGS.name())
+                                              .message("Custom permissions have been changed.")
+                                              .build());
+
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java

@@ -11,6 +11,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -23,7 +24,11 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.model.websocket.DossierEventType;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
 
+import com.iqser.red.service.persistence.management.v1.processor.service.FilterByPermissionsService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.JsonNode;
 import org.apache.commons.lang3.StringUtils;
+
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -76,6 +81,7 @@ public class DossierController implements DossierResource {
 
     private final DossierManagementService dossierManagementService;
     private final UserService userService;
+    private final FilterByPermissionsService filterByPermissionsService;
     private final FileStatusManagementService fileStatusManagementService;
     private final AuditPersistenceService auditPersistenceService;
     private final NotificationPersistenceService notificationPersistenceService;
@@ -106,6 +112,20 @@ public class DossierController implements DossierResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
+    public DossierChangeResponseV2 changesSinceV2(@RequestBody JSONPrimitive<OffsetDateTime> since) {
+
+        DossierChangeResponseV2 changes = dossierManagementService.changesSinceV2(since);
+
+        // filter only viewables
+        changes.setFileChanges(filterByPermissionsService.onlyViewableHavingDossierId(changes.getFileChanges()));
+        changes.setDossierChanges(filterByPermissionsService.onlyViewableHavingDossierId(changes.getDossierChanges()));
+
+        return changes;
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DOSSIER + "') && (#dossierRequest.dossierId == null || hasPermission(#dossierRequest.dossierId, 'Dossier', 'ACCESS_OBJECT') )")
     public ResponseEntity<Dossier> createDossierOrUpdateDossier(@RequestBody DossierRequest dossierRequest) {
@@ -405,6 +425,28 @@ public class DossierController implements DossierResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
+    public JSONPrimitive<Map<String, Dossier>> getDossiersByIds(@RequestBody JSONPrimitive<Set<String>> dossierIds) {
+
+        // filter dossiers based on view
+        var viewableDossierIds = filterByPermissionsService.onlyViewableDossierIds(dossierIds.getValue());
+        // load dossiers
+        var dossiers = dossierManagementService.getDossiersByIds(viewableDossierIds);
+
+        // add attributes and ACL - already filtered before loading
+        enhanceDossiersWithAttributeAndACLData(dossiers,false);
+
+        // build response
+        var responseMap = new LinkedHashMap<String, Dossier>();
+        for (var dossier : dossiers) {
+            responseMap.put(dossier.getId(), dossier);
+        }
+
+        return new JSONPrimitive<>(responseMap);
+    }
+
+
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
     public Dossier getDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
                               @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
@@ -418,70 +460,45 @@ public class DossierController implements DossierResource {
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                                      @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted) {
 
-        var dossiers = dossierManagementService.getAllDossiers(includeArchived, includeDeleted)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getAllDossiers(includeArchived, includeDeleted);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getDossiersForDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
                                                        @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                                                        @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted) {
 
-        var dossiers = dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getSoftDeletedDossiers() {
 
-        var dossiers =  dossierManagementService.getSoftDeletedDossiers()
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getSoftDeletedDossiers();
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiers() {
 
-        var dossiers =  dossierManagementService.getArchivedDossiers()
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getArchivedDossiers();
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiersForDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) {
 
-        var dossiers = dossierManagementService.getArchivedDossiersForDossierTemplateId(dossierTemplateId)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getArchivedDossiersForDossierTemplateId(dossierTemplateId);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
@@ -586,5 +603,31 @@ public class DossierController implements DossierResource {
         return new DossierAttributes(attributeIdToValue);
     }
 
+    private List<Dossier> enhanceDossiersWithAttributeAndACLData(List<Dossier> dossiers) {
+
+            return enhanceDossiersWithAttributeAndACLData(dossiers, true);
+    }
+
+    private List<Dossier> enhanceDossiersWithAttributeAndACLData(List<Dossier> dossiers, boolean filter) {
+
+        // filter first, only load attributes and ACL for viewable dossiers
+        List<Dossier> filteredDossiers = filter ? filterByPermissionsService.onlyViewableDossiers(dossiers) : dossiers;
+
+        // load all attributes at once
+        var attributes = dossierAttributePersistenceService.getDossierAttributes(filteredDossiers.stream().map(Dossier::getId).collect(Collectors.toSet()));
+        var attributesMap = new HashMap<String, List<DossierAttributeEntity>>();
+        for (DossierAttributeEntity attribute : attributes) {
+            attributesMap.computeIfAbsent(attribute.getId().getDossierId(), k -> new ArrayList<>()).add(attribute);
+        }
+
+        for (var dossier : filteredDossiers) {
+            // set attributes
+            dossier.setDossierAttributes(convertDossierAttributes(attributesMap.getOrDefault(dossier.getId(), new ArrayList<>())));
+            // set ACL data
+            dossierACLService.enhanceDossierWithACLData(dossier);
+        }
+        return filteredDossiers;
+    }
+
 }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierStatusController.java

@@ -14,13 +14,17 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.ColorUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.DossierStatusResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -31,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class DossierStatusController implements DossierStatusResource {
 
     private final DossierStatusPersistenceService dossierStatusPersistenceService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
@@ -57,6 +62,15 @@ public class DossierStatusController implements DossierStatusResource {
                                                                                            .rank(dossierStatusRequest.getRank())
                                                                                            .color(dossierStatusRequest.getColor())
                                                                                            .build());
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(dossierStatusRequest.getDossierTemplateId())
+                                              .category(AuditCategory.DOSSIER_TEMPLATE.name())
+                                              .message("Dossier states have been updated.")
+                                              .build());
+
+
         return MagicConverter.convert(response, DossierStatusInfo.class);
     }
 
@@ -91,7 +105,17 @@ public class DossierStatusController implements DossierStatusResource {
     public void deleteDossierStatus(@PathVariable("dossierStatusId") String dossierStatusId,
                                     @RequestParam(value = DOSSIER_STATUS_REPLACE_ID, required = false) String replaceDossierStatusId) {
 
+        var dossierTemplateId = dossierStatusPersistenceService.getDossierStatus(dossierStatusId).getDossierTemplateId();
+
         dossierStatusPersistenceService.deleteDossierStatus(dossierStatusId, replaceDossierStatusId);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(dossierTemplateId)
+                                              .category(AuditCategory.DOSSIER_TEMPLATE.name())
+                                              .message("Dossier state has been deleted.")
+                                              .build());
+
     }
 
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java

@@ -161,7 +161,7 @@ public class DownloadController implements DownloadResource {
         } // otherwise consider the files from dossier
 
         var validFilesAndNotProcessed = validFiles.stream()
-                .filter(f -> !(f.getAnalysisVersion() > 0 && f.getNumberOfAnalyses() > 0))
+                .filter(f -> !(f.getAnalysisVersion() > 0 && f.getNumberOfAnalyses() > 0 && !f.isSoftOrHardDeleted()))
                 .collect(Collectors.toList());
         if (!validFilesAndNotProcessed.isEmpty()) {
             throw new BadRequestException("At least a file is in its initial analysis process");

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileAttributesController.java

@@ -61,9 +61,9 @@ public class FileAttributesController implements FileAttributesResource {
         }
         fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
                                                                              MagicConverter.convert(fileAttributesConfig, FileAttributesGeneralConfigurationEntity.class));
-        var result = fileAttributeConfigPersistenceService.setFileAttributesConfig(dossierTemplateId,
-                                                                                   MagicConverter.convert(fileAttributesConfig.getFileAttributeConfigs(),
-                                                                                                          FileAttributeConfigEntity.class));
+        fileAttributeConfigPersistenceService.setFileAttributesConfig(dossierTemplateId,
+                                                                      MagicConverter.convert(fileAttributesConfig.getFileAttributeConfigs(), FileAttributeConfigEntity.class));
+
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierTemplateId)
@@ -74,7 +74,7 @@ public class FileAttributesController implements FileAttributesResource {
                 .filenameMappingColumnHeaderName(fileAttributesConfig.getFilenameMappingColumnHeaderName())
                 .delimiter(fileAttributesConfig.getDelimiter())
                 .encoding(fileAttributesConfig.getEncoding())
-                .fileAttributeConfigs(MagicConverter.convert(result, FileAttributeConfig.class))
+                .fileAttributeConfigs(MagicConverter.convert(fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId), FileAttributeConfig.class))
                 .build();
 
     }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java

@@ -8,6 +8,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -20,6 +21,7 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.mapper.EntityLogResponseMapper;
 import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
 import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
@@ -90,6 +92,7 @@ public class ManualRedactionController implements ManualRedactionResource {
     DictionaryPersistenceService dictionaryPersistenceService;
 
     EntityLogController entityLogController;
+    EntityLogResponseMapper mapper = EntityLogResponseMapper.INSTANCE;
 
 
     @PreAuthorize("hasAuthority('" + DELETE_MANUAL_REDACTION + "')")
@@ -252,8 +255,7 @@ public class ManualRedactionController implements ManualRedactionResource {
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse removeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                        @PathVariable(FILE_ID) String fileId,
-                                                       @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests,
-                                                       @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                       @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests) {
 
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
         verifyAccessForDossier(dossierId,
@@ -261,11 +263,7 @@ public class ManualRedactionController implements ManualRedactionResource {
                                removeRedactionRequests.stream()
                                        .anyMatch(RemoveRedactionRequestModel::isRemoveFromAllDossiers));
 
-        List<ManualAnnotationResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId,
-                                                                                                fileId,
-                                                                                                removeRedactionRequests,
-                                                                                                dossier.getDossierTemplateId(),
-                                                                                                includeUnprocessed);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId, fileId, removeRedactionRequests, dossier.getDossierTemplateId(), true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -280,10 +278,7 @@ public class ManualRedactionController implements ManualRedactionResource {
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
-    public ManualRedactionResponse removeRedactionBulkLocal(String dossierId,
-                                                            String fileId,
-                                                            RemoveRedactionBulkLocalRequestModel removeRedactionRequest,
-                                                            boolean includeUnprocessed) {
+    public ManualRedactionResponse removeRedactionBulkLocal(String dossierId, String fileId, RemoveRedactionBulkLocalRequestModel removeRedactionRequest) {
 
         verifyAccess(dossierId, fileId);
         verifyRequest(removeRedactionRequest.isRectangle(), removeRedactionRequest.getPosition(), removeRedactionRequest.getValue());
@@ -291,38 +286,36 @@ public class ManualRedactionController implements ManualRedactionResource {
         Set<RemoveRedactionRequestModel> removeRedactionRequestModels;
         FileModel status = fileStatusService.getStatus(fileId);
 
+        Set<EntityLogEntry> entries;
         if (!status.isExcludedFromAutomaticAnalysis()) {
-            Set<EntityLogEntry> entries = getFilteredEntityLogEntries(dossierId,
-                                                                      fileId,
-                                                                      removeRedactionRequest.isRectangle(),
-                                                                      removeRedactionRequest.getValue(),
-                                                                      removeRedactionRequest.isCaseSensitive(),
-                                                                      removeRedactionRequest.getOriginTypes(),
-                                                                      removeRedactionRequest.getOriginLegalBases(),
-                                                                      removeRedactionRequest.getPageNumbers(),
-                                                                      removeRedactionRequest.getPosition());
-            removeRedactionRequestModels = entries.stream()
-                    .map(entry -> RemoveRedactionRequestModel.builder().annotationId(entry.getId()).comment(removeRedactionRequest.getComment()).build())
-                    .collect(Collectors.toSet());
+            entries = getFilteredEntityLogEntries(dossierId,
+                                                  fileId,
+                                                  removeRedactionRequest.isRectangle(),
+                                                  removeRedactionRequest.getValue(),
+                                                  removeRedactionRequest.isCaseSensitive(),
+                                                  removeRedactionRequest.getOriginTypes(),
+                                                  removeRedactionRequest.getOriginLegalBases(),
+                                                  removeRedactionRequest.getPageNumbers(),
+                                                  removeRedactionRequest.getPosition());
 
         } else {
 
-            List<EntityLogEntryResponse> filteredEntityLogResponses = getFilteredEntityLogResponses(dossierId,
-                                                                                                    fileId,
-                                                                                                    includeUnprocessed,
-                                                                                                    removeRedactionRequest.isRectangle(),
-                                                                                                    removeRedactionRequest.getValue(),
-                                                                                                    removeRedactionRequest.isCaseSensitive(),
-                                                                                                    removeRedactionRequest.getOriginTypes(),
-                                                                                                    removeRedactionRequest.getOriginLegalBases(),
-                                                                                                    removeRedactionRequest.getPageNumbers(),
-                                                                                                    removeRedactionRequest.getPosition());
-
-            removeRedactionRequestModels = filteredEntityLogResponses.stream()
-                    .map(entityLogEntry -> RemoveRedactionRequestModel.builder().annotationId(entityLogEntry.getId()).comment(removeRedactionRequest.getComment()).build())
-                    .collect(Collectors.toSet());
+            entries = new HashSet<>(mapper.fromLogEntryResponses(getFilteredEntityLogResponses(dossierId,
+                                                                                               fileId,
+                                                                                               removeRedactionRequest.isRectangle(),
+                                                                                               removeRedactionRequest.getValue(),
+                                                                                               removeRedactionRequest.isCaseSensitive(),
+                                                                                               removeRedactionRequest.getOriginTypes(),
+                                                                                               removeRedactionRequest.getOriginLegalBases(),
+                                                                                               removeRedactionRequest.getPageNumbers(),
+                                                                                               removeRedactionRequest.getPosition())));
         }
-        return removeRedactionBulk(dossierId, fileId, removeRedactionRequestModels, includeUnprocessed);
+
+        removeRedactionRequestModels = entries.stream()
+                .map(entityLogEntry -> RemoveRedactionRequestModel.builder().annotationId(entityLogEntry.getId()).comment(removeRedactionRequest.getComment()).build())
+                .collect(Collectors.toSet());
+
+        return removeRedactionBulk(dossierId, fileId, removeRedactionRequestModels);
     }
 
 
@@ -371,13 +364,12 @@ public class ManualRedactionController implements ManualRedactionResource {
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse recategorizeBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                     @PathVariable(FILE_ID) String fileId,
-                                                    @RequestBody Set<RecategorizationRequestModel> recategorizationRequests,
-                                                    @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                    @RequestBody Set<RecategorizationRequestModel> recategorizationRequests) {
 
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
         verifyAccess(dossierId, fileId);
 
-        List<ManualAnnotationResponse> responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, includeUnprocessed);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -392,76 +384,61 @@ public class ManualRedactionController implements ManualRedactionResource {
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
-    public ManualRedactionResponse recategorizeBulkLocal(String dossierId,
-                                                         String fileId,
-                                                         RecategorizationBulkLocalRequestModel recategorizationRequest,
-                                                         boolean includeUnprocessed) {
+    public ManualRedactionResponse recategorizeBulkLocal(String dossierId, String fileId, RecategorizationBulkLocalRequestModel recategorizationRequest) {
 
         verifyAccess(dossierId, fileId);
         verifyRequest(recategorizationRequest.isRectangle(), recategorizationRequest.getPosition(), recategorizationRequest.getValue());
         Set<RecategorizationRequestModel> recategorizationRequestModels;
         FileModel status = fileStatusService.getStatus(fileId);
 
+        Set<EntityLogEntry> entries;
         if (!status.isExcludedFromAutomaticAnalysis()) {
-            Set<EntityLogEntry> entries = getFilteredEntityLogEntries(dossierId,
-                                                                      fileId,
-                                                                      recategorizationRequest.isRectangle(),
-                                                                      recategorizationRequest.getValue(),
-                                                                      recategorizationRequest.isCaseSensitive(),
-                                                                      recategorizationRequest.getOriginTypes(),
-                                                                      recategorizationRequest.getOriginLegalBases(),
-                                                                      recategorizationRequest.getPageNumbers(),
-                                                                      recategorizationRequest.getPosition());
-
-            recategorizationRequestModels = entries.stream()
-                    .map(entry -> RecategorizationRequestModel.builder()
-                            .annotationId(entry.getId())
-                            .type(recategorizationRequest.isRectangle() ? entry.getType() : recategorizationRequest.getType())
-                            .legalBasis(recategorizationRequest.getLegalBasis())
-                            .section(recategorizationRequest.getSection())
-                            .value(entry.getValue())
-                            .comment(recategorizationRequest.getComment())
-                            .build())
-                    .collect(Collectors.toSet());
+            entries = getFilteredEntityLogEntries(dossierId,
+                                                  fileId,
+                                                  recategorizationRequest.isRectangle(),
+                                                  recategorizationRequest.getValue(),
+                                                  recategorizationRequest.isCaseSensitive(),
+                                                  recategorizationRequest.getOriginTypes(),
+                                                  recategorizationRequest.getOriginLegalBases(),
+                                                  recategorizationRequest.getPageNumbers(),
+                                                  recategorizationRequest.getPosition());
 
         } else {
 
-            List<EntityLogEntryResponse> filteredEntityLogResponses = getFilteredEntityLogResponses(dossierId,
-                                                                                                    fileId,
-                                                                                                    includeUnprocessed,
-                                                                                                    recategorizationRequest.isRectangle(),
-                                                                                                    recategorizationRequest.getValue(),
-                                                                                                    recategorizationRequest.isCaseSensitive(),
-                                                                                                    recategorizationRequest.getOriginTypes(),
-                                                                                                    recategorizationRequest.getOriginLegalBases(),
-                                                                                                    recategorizationRequest.getPageNumbers(),
-                                                                                                    recategorizationRequest.getPosition());
-
-            recategorizationRequestModels = filteredEntityLogResponses.stream()
-                    .map(entityLogEntry -> RecategorizationRequestModel.builder()
-                            .annotationId(entityLogEntry.getId())
-                            .type(recategorizationRequest.isRectangle() ? entityLogEntry.getType() : recategorizationRequest.getType())
-                            .legalBasis(recategorizationRequest.getLegalBasis())
-                            .section(recategorizationRequest.getSection())
-                            .value(entityLogEntry.getValue())
-                            .comment(recategorizationRequest.getComment())
-                            .build())
-                    .collect(Collectors.toSet());
+            entries = new HashSet<>(mapper.fromLogEntryResponses(getFilteredEntityLogResponses(dossierId,
+                                                                                               fileId,
+                                                                                               recategorizationRequest.isRectangle(),
+                                                                                               recategorizationRequest.getValue(),
+                                                                                               recategorizationRequest.isCaseSensitive(),
+                                                                                               recategorizationRequest.getOriginTypes(),
+                                                                                               recategorizationRequest.getOriginLegalBases(),
+                                                                                               recategorizationRequest.getPageNumbers(),
+                                                                                               recategorizationRequest.getPosition())));
         }
 
-        return recategorizeBulk(dossierId, fileId, recategorizationRequestModels, includeUnprocessed);
+        recategorizationRequestModels = entries.stream()
+                .map(entry -> RecategorizationRequestModel.builder()
+                        .annotationId(entry.getId())
+                        .type(recategorizationRequest.isRectangle() ? entry.getType() : recategorizationRequest.getType())
+                        .legalBasis(recategorizationRequest.getLegalBasis())
+                        .section(recategorizationRequest.getSection())
+                        .value(recategorizationRequest.isRectangle() ? recategorizationRequest.getValue() : entry.getValue())
+                        .comment(recategorizationRequest.getComment())
+                        .build())
+                .collect(Collectors.toSet());
+
+        return recategorizeBulk(dossierId, fileId, recategorizationRequestModels);
     }
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse resizeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                        @PathVariable(FILE_ID) String fileId,
-                                                       @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests,
-                                                       @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                       @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests) {
 
         verifyAccessAndDossierExistence(dossierId, fileId);
 
-        List<ManualAnnotationResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, includeUnprocessed);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -497,7 +474,6 @@ public class ManualRedactionController implements ManualRedactionResource {
 
     private List<EntityLogEntryResponse> getFilteredEntityLogResponses(String dossierId,
                                                                        String fileId,
-                                                                       boolean includeUnprocessed,
                                                                        boolean rectangle,
                                                                        String value,
                                                                        boolean caseSensitive,
@@ -506,7 +482,7 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                                        Set<Integer> pageNumbers,
                                                                        Position position) {
 
-        List<EntityLogEntryResponse> entityLogEntryResponses = entityLogController.getEntityLog(dossierId, fileId, Collections.emptyList(), includeUnprocessed).getEntityLogEntry()
+        List<EntityLogEntryResponse> entityLogEntryResponses = entityLogController.getEntityLog(dossierId, fileId, Collections.emptyList(), true).getEntityLogEntry()
                 .stream()
                 .filter(entityLogEntryResponse -> !entityLogEntryResponse.getState().equals(EntryState.PENDING))
                 .collect(Collectors.toList());

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java

@@ -1,111 +0,0 @@
-package com.iqser.red.persistence.service.v1.external.api.impl.controller;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
-import com.iqser.red.service.persistence.service.v1.api.external.resource.MigrationStatusResource;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*;
-
-@RestController
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@RequiredArgsConstructor
-public class MigrationStatusController implements MigrationStatusResource {
-
-    SaasMigrationService saasMigrationService;
-
-    SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
-
-    FileStatusService fileStatusService;
-
-
-    public MigrationStatusResponse migrationStatus() {
-
-        int numberOfFilesToMigrate = saasMigrationStatusPersistenceService.countAll();
-
-        Map<SaasMigrationStatus, Integer> filesInStatus = new HashMap<>();
-        filesInStatus.put(MIGRATION_REQUIRED, saasMigrationStatusPersistenceService.countByStatus(MIGRATION_REQUIRED));
-        filesInStatus.put(DOCUMENT_FILES_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(DOCUMENT_FILES_MIGRATED));
-        filesInStatus.put(REDACTION_LOGS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(REDACTION_LOGS_MIGRATED));
-        filesInStatus.put(ANNOTATION_IDS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(ANNOTATION_IDS_MIGRATED));
-        filesInStatus.put(FINISHED, saasMigrationStatusPersistenceService.countByStatus(FINISHED));
-        filesInStatus.put(ERROR, saasMigrationStatusPersistenceService.countByStatus(ERROR));
-
-        var filesInErrorState = saasMigrationStatusPersistenceService.findAllByStatus(ERROR);
-
-        var errorCauses = filesInErrorState.stream()
-                .collect(Collectors.toMap(errorFile -> errorFile.getDossierId() + "/" + errorFile.getFileId(), SaasMigrationStatusEntity::getErrorCause));
-
-        return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> startMigrationForFile(String dossierId, String fileId) {
-
-        if (!fileStatusService.fileExists(fileId)) {
-            throw new NotFoundException(String.format("File with id %s does not exist", fileId));
-        }
-
-        saasMigrationService.startMigrationForFile(dossierId, fileId);
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> revertMigrationForFile(String dossierId, String fileId) {
-
-        if (!fileStatusService.fileExists(fileId)) {
-            throw new NotFoundException(String.format("File with id %s does not exist", fileId));
-        }
-
-        if (!saasMigrationStatusPersistenceService.findById(fileId).getStatus().equals(FINISHED)) {
-            throw new BadRequestException(String.format("File with id %s is not migrated yet, can't revert.", fileId));
-        }
-
-        saasMigrationService.revertMigrationForFile(dossierId, fileId);
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> requeueErrorFiles() {
-
-        MigrationStatusResponse migrationStatus = migrationStatus();
-        if (!migrationIsFinished(migrationStatus)) {
-            throw new BadRequestException("There are still files processing, please wait until migration has finished to retry!");
-        }
-
-        saasMigrationService.requeueErrorFiles();
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    private static boolean migrationIsFinished(MigrationStatusResponse migrationStatus) {
-
-        return migrationStatus.getFilesInStatus().entrySet()
-                .stream()
-                .filter(e -> e.getValue() > 0)
-                .allMatch(e -> e.getKey().equals(FINISHED) || e.getKey().equals(ERROR));
-    }
-
-}

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java

@@ -118,11 +118,12 @@ public class ReanalysisController implements ReanalysisResource {
     @PreAuthorize("hasAuthority('" + REANALYZE_FILE + "')")
     public void ocrFile(@PathVariable(DOSSIER_ID) String dossierId,
                         @PathVariable(FILE_ID) String fileId,
-                        @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force) {
+                        @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force,
+                        @RequestParam(value = ALL_PAGES, required = false, defaultValue = FALSE) boolean allPages) {
 
         accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
         validateOCR(dossierId, fileId);
-        reanalysisService.ocrFile(dossierId, fileId, force);
+        reanalysisService.ocrFile(dossierId, fileId, force, allPages);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierId)
@@ -140,7 +141,7 @@ public class ReanalysisController implements ReanalysisResource {
 
         accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
         fileIds.forEach(fileId -> validateOCR(dossierId, fileId));
-        reanalysisService.ocrFiles(dossierId, fileIds);
+        reanalysisService.ocrFiles(dossierId, fileIds, false);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierId)

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RulesController.java

@@ -26,6 +26,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.FileU
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.RulesValidationMapper;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.RulesResource;
@@ -52,6 +53,7 @@ public class RulesController implements RulesResource {
     private final RulesPersistenceService rulesPersistenceService;
     private final RulesValidationService rulesValidationService;
     private final AuditPersistenceService auditPersistenceService;
+    private final FileStatusPersistenceService fileStatusPersistenceService;
 
 
     @Override
@@ -75,6 +77,7 @@ public class RulesController implements RulesResource {
         }
         if (!rules.isDryRun()) {
             rulesPersistenceService.setRules(rulesUploadRequest.getRules(), rulesUploadRequest.getDossierTemplateId(), rulesUploadRequest.getRuleFileType());
+            fileStatusPersistenceService.resetErrorCounter(rules.getDossierTemplateId());
         }
 
         auditPersistenceService.audit(AuditRequest.builder()
@@ -157,4 +160,12 @@ public class RulesController implements RulesResource {
         return new ResponseEntity<>(new InputStreamResource(is), httpHeaders, HttpStatus.OK);
     }
 
+
+    @Override
+    @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
+    public void unlockRules(String dossierTemplateId, RuleFileType ruleFileType) {
+
+        rulesPersistenceService.resetTimeoutDetected(dossierTemplateId, ruleFileType);
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java

@@ -14,15 +14,21 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
 import org.springframework.security.access.AccessDeniedException;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
 import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
@@ -30,6 +36,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Approva
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
+import com.iqser.red.service.persistence.management.v1.processor.service.FilterByPermissionsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
@@ -39,6 +46,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
@@ -46,6 +54,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.notificatio
 import com.iqser.red.service.persistence.service.v1.api.shared.model.warning.ApproveResponse;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -67,6 +78,7 @@ public class StatusController implements StatusResource {
     private final NotificationPersistenceService notificationPersistenceService;
     private final DossierACLService dossierACLService;
     private final ApprovalVerificationService approvalVerificationService;
+    private final FilterByPermissionsService filterByPermissionsService;
 
 
     @Override
@@ -82,6 +94,28 @@ public class StatusController implements StatusResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
+    public JSONPrimitive<Map<String, List<FileStatus>>> getFilesByIds(@RequestBody JSONPrimitive<Map<String, Set<String>>> filesByDossier) {
+
+        // filter dossiers by view
+        var accessibleDossierIds = filterByPermissionsService.onlyViewableDossierIds(new ArrayList<>(filesByDossier.getValue().keySet()));
+        var response = new HashMap<String, List<FileStatus>>();
+        for (var dossierId : accessibleDossierIds) {
+            var allFoundFiles = fileStatusManagementService.findAllDossierIdAndIds(dossierId,
+                                                                                   filesByDossier.getValue()
+                                                                                           .get(dossierId));
+            response.put(dossierId,
+                         allFoundFiles.stream()
+                                 .map(FileStatusMapper::toFileStatus)
+                                 .collect(Collectors.toList()));
+        }
+
+        return new JSONPrimitive<>(response);
+
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
     public Map<String, List<FileStatus>> getDossierStatus(@RequestBody List<String> dossierIds) {
@@ -323,6 +357,10 @@ public class StatusController implements StatusResource {
                                                   .build());
 
             var dossier = dossierACLService.enhanceDossierWithACLData(dossierManagementService.getDossierById(dossierId, false, false));
+
+            if (dossier.getOwnerId() == null) {
+                throw new ConflictException("Dossier has no owner!");
+            }
             if (!dossier.getOwnerId().equals(KeycloakSecurity.getUserId())) {
 
                 var fileStatus = fileStatusManagementService.getFileStatus(fileId);
@@ -369,8 +407,10 @@ public class StatusController implements StatusResource {
 
     private void generatePossibleUnassignedFromFileNotification(String dossierId, String fileId, FileModel oldFileStatus, String newAssigneeId) {
 
-        if (oldFileStatus.getAssignee() == null || newAssigneeId == null || oldFileStatus.getAssignee().equals(newAssigneeId) || KeycloakSecurity.getUserId()
-                .equals(oldFileStatus.getAssignee())) {
+        if (oldFileStatus.getAssignee() == null
+            || newAssigneeId == null && oldFileStatus.getAssignee() == null
+            || oldFileStatus.getAssignee().equals(newAssigneeId)
+            || KeycloakSecurity.getUserId().equals(oldFileStatus.getAssignee())) {
             return;
         }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -21,6 +21,7 @@ import org.springframework.web.multipart.MultipartFile;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService;
+import com.iqser.red.service.persistence.management.v1.processor.migration.MigrationController;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
@@ -156,6 +157,7 @@ public class SupportController implements SupportResource {
         return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
     }
 
+
     @Override
     @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
     public ImportResponse importDataset(MultipartFile file) {

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java

@@ -269,7 +269,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     @Override
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
-    public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter) {
+    public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter, String quoteChar) {
 
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
@@ -285,18 +285,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
             throw new BadRequestException(format("The provided file name \"%s\" is not valid!", nameToUse));
         }
 
-        if (Strings.isNullOrEmpty(delimiter)) {
-            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
-        } else if (delimiter.length() != 1) {
-            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
-        }
-        char cleanDelimiter = delimiter.charAt(0);
+        char cleanDelimiter = getDelimiter(delimiter);
+        char cleanQuoteChar = getQuoteChar(quoteChar);
 
         Path mappingFile = saveToFile(file);
 
         try {
 
-            ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, cleanDelimiter, encoding, mappingFile.toFile());
+            ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
+                                                                               nameToUse,
+                                                                               fileName,
+                                                                               cleanDelimiter,
+                                                                               encoding,
+                                                                               mappingFile.toFile(),
+                                                                               cleanQuoteChar);
 
             return componentMappingMapper.toModel(metaData);
         } finally {
@@ -309,18 +311,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     @Override
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
-    public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String name, String encoding, String delimiter) {
+    public ComponentMappingMetadataModel updateMapping(String dossierTemplateId,
+                                                       String componentMappingId,
+                                                       MultipartFile file,
+                                                       String name,
+                                                       String encoding,
+                                                       String delimiter,
+                                                       String quoteChar) {
 
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
         String nameToUse = validateFileName(file, name);
 
-        if (Strings.isNullOrEmpty(delimiter)) {
-            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
-        } else if (delimiter.length() != 1) {
-            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
-        }
-        char cleanDelimiter = delimiter.charAt(0);
+        char cleanDelimiter = getDelimiter(delimiter);
+        char cleanQuoteChar = getQuoteChar(quoteChar);
 
         Path mappingFile = saveToFile(file);
 
@@ -331,7 +335,8 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
                                                                                      encoding,
                                                                                      cleanDelimiter,
                                                                                      mappingFile.toFile(),
-                                                                                     file.getOriginalFilename());
+                                                                                     file.getOriginalFilename(),
+                                                                                     cleanQuoteChar);
 
             return componentMappingMapper.toModel(resultMetaData);
         } finally {
@@ -340,6 +345,28 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     }
 
 
+    private static char getDelimiter(String delimiter) {
+
+        if (Strings.isNullOrEmpty(delimiter)) {
+            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
+        } else if (delimiter.length() != 1) {
+            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
+        }
+        return delimiter.charAt(0);
+    }
+
+
+    private static char getQuoteChar(String quoteChar) {
+
+        if (Strings.isNullOrEmpty(quoteChar)) {
+            throw new BadRequestException("The provided quoteChar is not valid! Can't be null or empty.");
+        } else if (quoteChar.length() != 1) {
+            throw new BadRequestException(format("The provided quoteChar %s is not valid! Only a single character is allowed.", quoteChar));
+        }
+        return quoteChar.charAt(0);
+    }
+
+
     private static String validateFileName(MultipartFile file, String name) {
 
         if (Strings.isNullOrEmpty(file.getOriginalFilename()) || !file.getOriginalFilename().endsWith(".csv")) {

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java

@@ -226,7 +226,6 @@ public interface DictionaryResource {
                      @RequestParam(value = "addToDictionary") boolean addToDictionary);
 
 
-    @ResponseStatus(HttpStatus.ACCEPTED)
     @PostMapping(value = DICTIONARY_REST_PATH + DIFFERENCE + DOSSIER_TEMPLATE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Returns the difference between the dictionaries of the dossier template and all the dossiers inside the template for a list of given types.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Successfully returned DictionaryDifferenceResponse."), @ApiResponse(responseCode = "400", description = "The request is not valid.")})

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DossierResource.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.external.resource;
 
 import java.time.OffsetDateTime;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.springframework.http.HttpStatus;
@@ -17,6 +18,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierInformation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
@@ -29,10 +31,12 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
 public interface DossierResource {
 
     String DOSSIER_REST_PATH = ExternalApi.BASE_PATH + "/dossier";
+    String BY_ID_PATH = "/by-id";
     String DOSSIER_TEMPLATE_PATH = "/dossier-template";
     String DOSSIER_INFO_PATH = "/info";
     String DELETED_DOSSIERS_PATH = ExternalApi.BASE_PATH + "/deleted-dossiers";
     String CHANGES_DETAILS_PATH = "/changes/details";
+    String CHANGES_DETAILS_V2_PATH = "/changes/details/v2";
     String HARD_DELETE_PATH = "/hard-delete";
     String UNDELETE_PATH = "/restore";
 
@@ -62,6 +66,12 @@ public interface DossierResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success")})
     List<DossierChangeEntry> changesSince(@RequestBody JSONPrimitive<OffsetDateTime> since);
 
+    @ResponseBody
+    @PostMapping(value = DOSSIER_REST_PATH + CHANGES_DETAILS_V2_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "See if there are changes to dossiers since param", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success")})
+    DossierChangeResponseV2 changesSinceV2(@RequestBody JSONPrimitive<OffsetDateTime> since);
+
 
     @ResponseBody
     @PostMapping(value = DOSSIER_REST_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@@ -95,6 +105,13 @@ public interface DossierResource {
     List<Dossier> getDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                               @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted);
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = DOSSIER_REST_PATH+BY_ID_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets dossiers by ids.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    JSONPrimitive<Map<String,Dossier>> getDossiersByIds(@RequestBody JSONPrimitive<Set<String>> dossierIds);
+
 
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DossierTemplateResource.java

@@ -95,35 +95,36 @@ public interface DossierTemplateResource {
 
     @ResponseStatus(value = HttpStatus.NO_CONTENT)
     @PostMapping(value = DOSSIER_TEMPLATE_PATH + IMPORT_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Receives an archive to import", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Archive have successfully imported"), @ApiResponse(responseCode = "400", description = "Import process stuck in one of the steps:"
-                                                                                                                                                                    + "0 - Reading the archive content\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "1 - store information about the dossier template\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "2 - store the colors\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "3 - store the watermarks\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "4 - store the dossier status\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "5 - store the dossier attributes\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "6 - store the file attributes\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "7 - store the report templates\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "8 - store the legal basis\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "9 - store the file attribute configuration\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "10 - store the component definitions\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "11 - store the component mappings\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "12 - store the types and entities\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "13 - store the rules and component rules"), @ApiResponse(responseCode = "404", description = "The dossier template to update does not exist")})
+    @Operation(summary = "Receives an archive to import", description = """
+            Import process stuck in one of the steps:"
+            0 - Reading the archive content
+            
+            1 - store information about the dossier template
+            
+            2 - store the colors
+            
+            3 - store the watermarks
+            
+            4 - store the dossier status
+            
+            5 - store the dossier attributes
+            
+            6 - store the file attributes
+            
+            7 - store the report templates
+            
+            8 - store the legal basis
+            
+            9 - store the file attribute configuration
+            
+            10 - store the component definitions
+            
+            11 - store the component mappings
+            
+            12 - store the types and entities
+            
+            13 - store the rules and component rules""")
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Archive have successfully imported"), @ApiResponse(responseCode = "400"), @ApiResponse(responseCode = "404", description = "The dossier template to update does not exist")})
     DossierTemplateModel importDossierTemplate(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                @RequestParam(value = DOSSIER_TEMPLATE_ID, required = false) String dossierTemplateId,
                                                @RequestParam(value = "updateExistingDossierTemplate", required = false, defaultValue = "false") boolean updateExistingDossierTemplate);

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/FileAttributesResource.java

@@ -44,7 +44,11 @@ public interface FileAttributesResource {
     String FILE_ID = "fileId";
     String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
 
-    Set<String> encodingList = Sets.newHashSet("ISO", "ASCII", "UTF-8");
+    String UTF_ENCODING = "UTF-8";
+    String ASCII_ENCODING = "ASCII";
+    String ISO_ENCODING = "ISO-8859-1";
+
+    Set<String> encodingList = Sets.newHashSet(ISO_ENCODING, ASCII_ENCODING, UTF_ENCODING);
 
 
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ManualRedactionResource.java

@@ -119,8 +119,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse removeRedactionBulkLocal(@PathVariable(DOSSIER_ID) String dossierId,
                                                      @PathVariable(FILE_ID) String fileId,
-                                                     @RequestBody RemoveRedactionBulkLocalRequestModel removeRedactionRequest,
-                                                     @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                     @RequestBody RemoveRedactionBulkLocalRequestModel removeRedactionRequest);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -132,8 +131,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse removeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                 @PathVariable(FILE_ID) String fileId,
-                                                @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests,
-                                                @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -170,8 +168,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse recategorizeBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                              @PathVariable(FILE_ID) String fileId,
-                                             @RequestBody Set<RecategorizationRequestModel> recategorizationRequests,
-                                             @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                             @RequestBody Set<RecategorizationRequestModel> recategorizationRequests);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -183,8 +180,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse recategorizeBulkLocal(@PathVariable(DOSSIER_ID) String dossierId,
                                                   @PathVariable(FILE_ID) String fileId,
-                                                  @RequestBody RecategorizationBulkLocalRequestModel recategorizationRequest,
-                                                  @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                  @RequestBody RecategorizationBulkLocalRequestModel recategorizationRequest);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -196,8 +192,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier  or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse resizeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                 @PathVariable(FILE_ID) String fileId,
-                                                @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests,
-                                                @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests);
 
 
     @ResponseStatus(value = HttpStatus.OK)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java

@@ -1,56 +0,0 @@
-package com.iqser.red.service.persistence.service.v1.api.external.resource;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
-
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
-
-public interface MigrationStatusResource {
-
-    String MIGRATION_STATUS_REST_PATH = ExternalApi.BASE_PATH + "/migration-status";
-    String START_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/start_migration";
-    String REVERT_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/revert_migration";
-    String RETRY_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/retry_migration";
-
-    String FILE_ID = "fileId";
-    String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
-
-    String DOSSIER_ID = "dossierId";
-    String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
-
-
-    @ResponseBody
-    @PostMapping(value = MIGRATION_STATUS_REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Show the status of the migration", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    MigrationStatusResponse migrationStatus();
-
-
-    @ResponseBody
-    @PostMapping(value = START_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
-    @Operation(summary = "Start SAAS migration for specific file", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> startMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
-
-
-    @ResponseBody
-    @PostMapping(value = REVERT_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
-    @Operation(summary = "Start SAAS migration for specific file", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> revertMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
-
-
-    @ResponseBody
-    @PostMapping(value = RETRY_MIGRATION_REST_PATH)
-    @Operation(summary = "Restart SAAS migration for all files in error state", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> requeueErrorFiles();
-
-}

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ReanalysisResource.java

@@ -38,6 +38,7 @@ public interface ReanalysisResource {
 
     String EXCLUDED_STATUS_PARAM = "excluded";
     String FORCE_PARAM = "force";
+    String ALL_PAGES = "allPages";
 
 
     @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE)
@@ -73,7 +74,8 @@ public interface ReanalysisResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "409", description = "Conflict"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden"), @ApiResponse(responseCode = "400", description = "Cannot OCR approved file")})
     void ocrFile(@PathVariable(DOSSIER_ID) String dossierId,
                  @PathVariable(FILE_ID) String fileId,
-                 @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force);
+                 @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force,
+                 @RequestParam(value = ALL_PAGES, required = false, defaultValue = FALSE) boolean allPages);
 
 
     @Operation(summary = "Ocr and reanalyze multiple files for a dossier", description = "None")

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RulesResource.java

@@ -6,6 +6,7 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RequestPart;
@@ -28,6 +29,7 @@ public interface RulesResource {
     String RULES_PATH = ExternalApi.BASE_PATH + "/rules";
     String UPLOAD_PATH = "/upload";
     String DOWNLOAD_PATH = "/download";
+    String RESET_PATH = "/reset";
 
     String DOSSIER_TEMPLATE_PARAMETER_NAME = "dossierTemplateId";
     String DOSSIER_TEMPLATE_PATH_VARIABLE = "/{dossierTemplateId}";
@@ -105,4 +107,10 @@ public interface RulesResource {
     @GetMapping(value = RULES_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE + RULE_FILE_TYPE_PATH_VARIABLE + DOWNLOAD_PATH)
     ResponseEntity<?> downloadFile(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType);
 
+    @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
+    @Operation(summary = "Resets the timeout detected flag in a Rule file.")
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "No content")})
+    @PutMapping(value = RULES_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE + RULE_FILE_TYPE_PATH_VARIABLE + RESET_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    void unlockRules(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType);
 }

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/StatusResource.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.external.resource;
 import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -16,6 +17,7 @@ import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.warning.ApproveResponse;
 
 import io.swagger.v3.oas.annotations.Operation;
@@ -25,6 +27,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
 public interface StatusResource {
 
     String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status";
+    String BY_ID_PATH = "/by-id";
     String CHANGES_SINCE_PATH = "/changes";
     String BULK_REST_PATH = "/bulk";
     String ASSIGNEE_REST_PATH = "/set-assignee";
@@ -56,6 +59,14 @@ public interface StatusResource {
     Map<String, List<FileStatus>> getDossierStatus(@RequestBody List<String> dossierIds);
 
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = STATUS_REST_PATH + BY_ID_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets the status for files by dossierId and fileIds.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    JSONPrimitive<Map<String, List<FileStatus>>> getFilesByIds(@RequestBody JSONPrimitive<Map<String, Set<String>>> filesByDossier);
+
+
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody
     @PostMapping(value = STATUS_REST_PATH + DELETED_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -146,9 +146,7 @@ public interface SupportResource {
 
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody
-    @PostMapping(value = DATASET_EXCHANGE
-                         + EXPORT
-                         + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = DATASET_EXCHANGE + EXPORT + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
             ## Export Preview Files Endpoint
             
@@ -166,6 +164,7 @@ public interface SupportResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
     ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
+
     @ResponseBody
     @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.service.v2.api.external.resource;
 import java.util.List;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DateFormatPatternErrorMessage;
 import com.iqser.red.service.persistence.service.v2.api.external.model.DossierAttributeDefinitionList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.DossierStatusDefinitionList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList;
@@ -63,6 +62,7 @@ public interface DossierTemplateResource {
     String DRY_RUN_PARAM = "dryRun";
     String ENCODING_PARAM = "encoding";
     String DELIMITER_PARAM = "delimiter";
+    String QUOTE_CHAR_PARAM = "quoteChar";
     String MAPPING_NAME_PARAM = "name";
 
     String INCLUDE_SOFT_DELETED = "includeSoftDeleted";
@@ -127,7 +127,7 @@ public interface DossierTemplateResource {
     @Operation(summary = "Upload a date formats file for a specific DossierTemplate.")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Date formats upload successful."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found."), @ApiResponse(responseCode = "400", description = "Uploaded date formats could not be verified."), @ApiResponse(responseCode = "422", description = "Uploaded date formats file could not be parsed.")})
     ResponseEntity<?> uploadDateFormats(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
-                                                                          @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
+                                        @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
 
     @ResponseBody
@@ -159,7 +159,8 @@ public interface DossierTemplateResource {
                                                 @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                 @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
                                                 @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
-                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
+                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
+                                                @Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
 
 
     @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None")
@@ -173,7 +174,8 @@ public interface DossierTemplateResource {
                                                 @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                 @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
                                                 @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
-                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
+                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
+                                                @Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
 
 
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml

@@ -509,7 +509,7 @@ paths:
         - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
         
         #### Customization Options
-        - Users can specify the delimiter and encoding used in the CSV file.
+        - Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
         
         #### Usage
         - The component mapping file can be utilized in component rules to relate components to existing master data.
@@ -533,6 +533,7 @@ paths:
         - $ref: '#/components/parameters/mappingName'
         - $ref: '#/components/parameters/encoding'
         - $ref: '#/components/parameters/delimiter'
+        - $ref: '#/components/parameters/quoteChar'
       responses:
         "200":
           content:
@@ -609,7 +610,7 @@ paths:
         - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
         
         #### Customization Options
-        - Users can specify the delimiter and encoding used in the CSV file.
+        - Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
       tags:
         - 1. Dossier Templates
       requestBody:
@@ -623,6 +624,7 @@ paths:
         - $ref: '#/components/parameters/mappingName'
         - $ref: '#/components/parameters/encoding'
         - $ref: '#/components/parameters/delimiter'
+        - $ref: '#/components/parameters/quoteChar'
       responses:
         "200":
           content:
@@ -2114,6 +2116,17 @@ components:
         example: ','
         default: ','
       description: "The delimiter used as a separator in a csv file."
+    quoteChar:
+      name: quoteChar
+      required: false
+      in: query
+      schema:
+        type: string
+        minLength: 1
+        maxLength: 1
+        example: '"'
+        default: '"'
+      description: "The quoteChar used to quote fields in a csv file."
     mappingName:
       name: name
       required: false

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java

@@ -57,7 +57,7 @@ public class AdminInterfaceController {
 
         fileStatusService.validateFileIsNotDeletedAndNotApproved(fileId);
 
-        fileStatusService.setStatusOcrQueued(dossierId, fileId);
+        fileStatusService.setStatusOcrQueued(dossierId, fileId, false);
 
     }
 
@@ -91,7 +91,7 @@ public class AdminInterfaceController {
 
                     if (!dryRun) {
                         fileStatusService.validateFileIsNotDeletedAndNotApproved(file.getId());
-                        fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId());
+                        fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId(), false);
                     }
 
                 }

persistence-service-v1/persistence-service-processor-v1/build.gradle.kts

@@ -38,10 +38,7 @@ dependencies {
         exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
         exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
     }
-    api("com.knecon.fforesight.service:ocr-service-api:4.24.0") {
-        exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
-        exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
-    }
+    api("com.knecon.fforesight:azure-ocr-service-api:0.13.0")
     implementation("com.knecon.fforesight:llm-service-api:1.17.0")
     api("com.knecon.fforesight:jobs-commons:0.10.0")
     api("com.iqser.red.commons:storage-commons:2.50.0")
@@ -78,8 +75,8 @@ dependencies {
     api("com.opencsv:opencsv:5.9")
 
     implementation("com.google.protobuf:protobuf-java:4.27.1")
-    implementation("org.mapstruct:mapstruct:1.5.5.Final")
-    annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final")
+    implementation("org.mapstruct:mapstruct:1.6.2")
+    annotationProcessor("org.mapstruct:mapstruct-processor:1.6.2")
 
     testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
     testImplementation("org.testcontainers:postgresql:1.17.1")

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/RedObjectIdentityRetrievalStrategy.java

@@ -5,6 +5,7 @@ import org.springframework.security.acls.model.ObjectIdentity;
 import org.springframework.security.acls.model.ObjectIdentityRetrievalStrategy;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.IHavingDossierId;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 
 import lombok.extern.slf4j.Slf4j;
@@ -18,8 +19,8 @@ public class RedObjectIdentityRetrievalStrategy implements ObjectIdentityRetriev
         log.debug("Requesting data for object: {}", domainObject);
         if (domainObject instanceof Dossier) {
             return new ObjectIdentityImpl("Dossier", ((Dossier) domainObject).getId());
-        } else if (domainObject instanceof DossierChangeEntry) {
-            return new ObjectIdentityImpl("Dossier", ((DossierChangeEntry) domainObject).getDossierId());
+        } else if (domainObject instanceof IHavingDossierId) {
+            return new ObjectIdentityImpl("Dossier", ((IHavingDossierId) domainObject).getDossierId());
         } else if (domainObject instanceof String) {
             // TODO ACL this will not work once we have more than one type.
             return new ObjectIdentityImpl("Dossier", (String) domainObject);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/FileExchangeImportModel.java

@@ -50,7 +50,8 @@ public class FileExchangeImportModel {
         EntityLog entityLog;
         ComponentLog componentLog;
         List<ComponentLogEntry> overrides;
-        ManualChangesExportModel manualChanges;
+        @Builder.Default
+        ManualChangesExportModel manualChanges = new ManualChangesExportModel();
         @Builder.Default
         List<FileAttribute> fileAttributes = new ArrayList<>();
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualAddExportModel.java

@@ -11,13 +11,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualChangesExportModel.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import lombok.AccessLevel;
@@ -16,11 +17,17 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class ManualChangesExportModel {
 
-    List<ManualAddExportModel> manualAddExportModels;
-    List<ManualRecategorizationExportModel> manualRecategorizationExportModels;
-    List<ManualResizeExportModel> manualResizeExportModels;
-    List<ManualForceExportModel> manualForceExportModels;
-    List<ManualLegalBasisChangeExportModel> manualLegalBasisChangeExportModels;
-    List<ManualRemoveExportModel> manualRemoveExportModels;
+    @Builder.Default
+    List<ManualAddExportModel> manualAddExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualRecategorizationExportModel> manualRecategorizationExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualResizeExportModel> manualResizeExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualForceExportModel> manualForceExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualLegalBasisChangeExportModel> manualLegalBasisChangeExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualRemoveExportModel> manualRemoveExportModels = new ArrayList<>();
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualForceExportModel.java

@@ -10,7 +10,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualLegalBasisChangeExportModel.java

@@ -8,7 +8,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class ManualLegalBasisChangeExportModel {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualRecategorizationExportModel.java

@@ -12,7 +12,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualRemoveExportModel.java

@@ -10,7 +10,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class ManualRemoveExportModel {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualResizeExportModel.java

@@ -14,7 +14,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierExportService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
 
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
@@ -9,6 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
@@ -33,29 +35,30 @@ public class DossierExportService {
     FileStatusManagementService fileStatusManagementService;
     EntityTypeExportService entityTypeExportService;
     ObjectMapper mapper;
+    FileStatusService fileStatusService;
 
 
     @SneakyThrows
     @Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
     public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
 
-        List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
+        List<String> fileIdsInDossier = fileStatusManagementService.getDossierStatusIds(dossier.getId(), false)
+                .stream()
+                .filter(fileId -> request.fileIds().isEmpty() || request.fileIds().contains(fileId))
+                .toList();
 
-        if (!request.dossierIds().contains(dossier.getId()) //
-            && files.stream()
-                    .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
-            // dossier has no files in requested files and dossier not explicitly requested -> don't export it.
+        if (fileIdsInDossier.isEmpty()) {
             return;
         }
 
         Path dossierFolder = folder.resolve(dossier.getId());
         archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
 
-        for (FileModel fileEntity : files) {
-            if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
-                continue;
-            }
-            fileExportService.addFileToArchive(archiver, dossierFolder, request, fileEntity);
+        for (String fileId : fileIdsInDossier) {
+
+            FileModel file = fileStatusService.getStatus(fileId);
+
+            fileExportService.addFileToArchive(archiver, dossierFolder, request, file);
         }
 
         List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java

@@ -275,8 +275,7 @@ public class DossierTemplateExportService {
         // and 1 txt file for every type: entries, false positives and false recommendation
 
         // remove the types related to dossiers and also the ones that are deleted
-        // also the ones that are system - managed
-        List<TypeEntity> dossierTypes = dictionaryPersistenceService.getAllTypesForDossierTemplateWithoutSystemManaged(dossierTemplate.getId(), false);
+        List<TypeEntity> dossierTypes = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplate.getId(), false);
         for (TypeEntity typeEntity : dossierTypes) {
 //                log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId() + " " + typeEntity.isDeleted());
             entityTypeExportService.addEntityTypeToArchive(fileSystemBackedArchiver, typeEntity, folder);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateImportService.java

@@ -41,6 +41,7 @@ import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zi
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DateFormatsValidationService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DefaultDateFormatsProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReportTemplateService;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
@@ -75,10 +76,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -109,6 +108,7 @@ public class DossierTemplateImportService {
     private final ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
     private final EntityTypeImportService entityTypeImportService;
     private final SystemManagedTypesImport systemManagedTypesImport;
+    private final DefaultDateFormatsProvider defaultDateFormatsProvider;
 
 
     public String importDossierTemplate(ImportDossierTemplateRequest request) {
@@ -285,7 +285,7 @@ public class DossierTemplateImportService {
                 importStep = 12;
                 entityTypeImportService.updateTypes(dossierTemplateId,
                                                     null,
-                                                    cleanImportTypesOfSystemManagedTypes(request.getEntityTypeImportModel(), systemManagedTypesImport.getSystemManagedTypeList()));
+                                                    entityTypeImportService.updateAndCleanSystemManagedFromImportTypes(request.getEntityTypeImportModel(), dossierTemplateId));
 
             } else {
 
@@ -402,15 +402,9 @@ public class DossierTemplateImportService {
 
                 //set types
                 importStep = 12;
-                // import system managed entity types
-                EntityTypeImportModel systemManagedTypesModel = new EntityTypeImportModel();
-                systemManagedTypesModel.getTypes().addAll(systemManagedTypesImport.getSystemManagedTypeList());
-                entityTypeImportService.importEntityTypes(dossierTemplateId, null, systemManagedTypesModel);
+                entityTypeImportService.addMissingSystemManagedTypesToImport(request.getEntityTypeImportModel(), systemManagedTypesImport.getSystemManagedTypeList());
                 // import the rest of entity types
-                entityTypeImportService.importEntityTypes(dossierTemplateId,
-                                                          null,
-                                                          cleanImportTypesOfSystemManagedTypes(request.getEntityTypeImportModel(),
-                                                                                               systemManagedTypesImport.getSystemManagedTypeList()));
+                entityTypeImportService.importEntityTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
 
             }
 
@@ -449,7 +443,8 @@ public class DossierTemplateImportService {
                                                                                      componentMapping.metadata().getFileName(),
                                                                                      componentMapping.metadata().getDelimiter(),
                                                                                      componentMapping.metadata().getEncoding(),
-                                                                                     tmpFile);
+                                                                                     tmpFile,
+                                                                                     componentMapping.metadata().getQuoteChar());
 
             componentMappingService.setVersion(createdMapping.getId(), componentMapping.metadata().getVersion());
             assert tmpFile.delete();
@@ -494,13 +489,18 @@ public class DossierTemplateImportService {
 
     private void setDataFormats(ImportTemplateResult request, String dossierTemplateId) {
 
-        if (request.getDateFormats() != null) {
-            List<DateFormatPatternErrorMessage> dateFormatPatternErrorMessages = dateFormatsValidationService.validateDateFormats(request.getDateFormats());
+        String dateFormats = request.getDateFormats();
+        if (dateFormats == null && applicationType.equals("DocuMine")) {
+            dateFormats = defaultDateFormatsProvider.getDateFormats();
+        }
+        if (dateFormats != null) {
+            List<DateFormatPatternErrorMessage> dateFormatPatternErrorMessages = dateFormatsValidationService.validateDateFormats(dateFormats);
             if (!dateFormatPatternErrorMessages.isEmpty()) {
                 throw new BadRequestException("The date formats file contains errors");
             }
-            dateFormatsPersistenceService.setDateFormats(request.dateFormats, dossierTemplateId, request.getDateFormatsExportModel().getVersion());
+            dateFormatsPersistenceService.setDateFormats(dateFormats, dossierTemplateId, request.getDateFormatsExportModel().getVersion());
         }
+
     }
 
 
@@ -524,6 +524,7 @@ public class DossierTemplateImportService {
 
     }
 
+
     private LayoutParsingType deferFromApplicationType() {
 
         return Objects.equals(applicationType, "DocuMine") ? LayoutParsingType.DOCUMINE_OLD : LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH;
@@ -627,6 +628,9 @@ public class DossierTemplateImportService {
 
     private void updateComponents(ImportTemplateResult request, String dossierTemplateId) {
 
+        List<ComponentDefinitionEntity> existingComponentDefinitions = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
+        existingComponentDefinitions.forEach(componentDefinition -> componentDefinitionPersistenceService.delete(componentDefinition.getId()));
+
         request.getComponentDefinitions()
                 .forEach(componentDefinition -> {
                     componentDefinition.setDossierTemplateId(dossierTemplateId);
@@ -670,30 +674,4 @@ public class DossierTemplateImportService {
         return dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
     }
 
-
-    private EntityTypeImportModel cleanImportTypesOfSystemManagedTypes(EntityTypeImportModel importModel, List<Type> systemManagedTypes) {
-
-        Set<String> systemManagedTypesByName = systemManagedTypes.stream()
-                .map(Type::getType)
-                .collect(Collectors.toSet());
-        Set<String> systemManagedTypesIdsFromImport = importModel.getTypes()
-                .stream()
-                .filter(t -> systemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
-                .map(Type::getType)
-                .collect(Collectors.toSet());
-
-        if (!systemManagedTypesIdsFromImport.isEmpty()) {
-
-            importModel.getTypes().removeIf(t -> systemManagedTypesIdsFromImport.contains(t.getType()));
-
-            importModel.getEntries().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getFalsePositives().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getFalseRecommendations().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getDeletedEntries().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getDeletedFalsePositives().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getDeletedFalseRecommendations().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-        }
-        return importModel;
-    }
-
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/EntityTypeImportService.java

@@ -129,6 +129,73 @@ public class EntityTypeImportService {
         }
     }
 
+    public void addMissingSystemManagedTypesToImport(EntityTypeImportModel importModel, List<Type> systemManagedTypes) {
+        Set<String> defaultSystemManagedTypesByName = systemManagedTypes.stream()
+                .map(Type::getType)
+                .collect(Collectors.toSet());
+        importModel.getTypes()
+                .stream()
+                .filter(t -> defaultSystemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
+                .forEach(t -> {
+                    if (!t.isSystemManaged()) {
+                        t.setSystemManaged(true);
+                    }
+                });
+
+        Set<String> systemManagedTypesFromImport = importModel.getTypes()
+                .stream()
+                .filter(t -> defaultSystemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
+                .map(Type::getType)
+                .collect(Collectors.toSet());
+        List<Type> systemManagedTypesNotIncludedInTheImport = systemManagedTypes.stream()
+                .filter(type -> !systemManagedTypesFromImport.contains(type.getType()))
+                .collect(Collectors.toList());
+        importModel.getTypes().addAll(systemManagedTypesNotIncludedInTheImport);
+    }
+
+    public EntityTypeImportModel updateAndCleanSystemManagedFromImportTypes(EntityTypeImportModel importModel, String dossierTemplateId) {
+
+        List<TypeEntity> currentSystemManagedTypes = dossierTemplatePersistenceService.getTypesForDossierTemplate(dossierTemplateId)
+                .stream()
+                .filter(t -> t.getDossierId() == null)
+                .filter(t -> t.isSystemManaged())
+                .collect(Collectors.toList());
+
+        var currentSystemManagedTypesNameToTypeMap = currentSystemManagedTypes.stream()
+                .collect(Collectors.toMap(TypeEntity::getType, Function.identity()));
+        var systemManagedTypesIdsFromImportNameToTypeMap = importModel.getTypes()
+                .stream()
+                .filter(t -> currentSystemManagedTypesNameToTypeMap.keySet().contains(t.getType()) || t.isSystemManaged())
+                .collect(Collectors.toMap(Type::getType, Function.identity()));
+
+        if (!systemManagedTypesIdsFromImportNameToTypeMap.isEmpty()) {
+            systemManagedTypesIdsFromImportNameToTypeMap.entrySet().forEach(entry -> {
+                var typeId = currentSystemManagedTypesNameToTypeMap.get(entry.getKey()).getId();
+                entry.getValue().setDossierTemplateId(dossierTemplateId);
+                entry.getValue().setDossierId(null);
+                dictionaryManagementService.updateTypeValue(typeId, entry.getValue());
+
+                this.addEntries(importModel.getEntries(), importModel.getDeletedEntries(), typeId, entry.getKey(), DictionaryEntryType.ENTRY);
+                this.addEntries(importModel.getFalsePositives(), importModel.getDeletedFalsePositives(), typeId, entry.getKey(), DictionaryEntryType.FALSE_POSITIVE);
+                this.addEntries(importModel.getFalseRecommendations(), importModel.getDeletedFalseRecommendations(), typeId, entry.getKey(), DictionaryEntryType.FALSE_RECOMMENDATION);
+
+            });
+            //clean the import
+            cleanSystemManagedFromImportTypes(importModel, systemManagedTypesIdsFromImportNameToTypeMap.keySet());
+        }
+        return importModel;
+    }
+
+    private void cleanSystemManagedFromImportTypes(EntityTypeImportModel importModel, Set<String> systemManagedTypesIdsFromImport) {
+        importModel.getTypes().removeIf(t -> systemManagedTypesIdsFromImport.contains(t.getType()));
+
+        importModel.getEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+    }
 
     private void enrichObservation(EntityTypeImportModel importModel) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java

@@ -3,11 +3,14 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
@@ -83,7 +86,7 @@ public class FileExchangeExportService {
 
 
     @Observed(name = "FileExchangeExportService", contextualName = "export-files")
-    public long createDownloadArchive(ExportDownloadMessage downloadJob) throws IOException {
+    public long createDownloadArchive(ExportDownloadMessage downloadJob) {
 
         FileExchangeExportRequest request = downloadJob.getFileExchangeExportRequest();
         DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(downloadJob.getDossierTemplateId());
@@ -92,13 +95,18 @@ public class FileExchangeExportService {
 
             dossierTemplateExportService.addDossierTemplateToArchive(archiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate);
 
-            for (Dossier dossierEntity : dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplate.getId(), true, false)) {
+            Iterable<String> dossierIds;
+            if (request.dossierIds().isEmpty()) {
+                dossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplate.getId(), true, false);
+            } else {
+                dossierIds = request.dossierIds();
+            }
 
-                if (!request.dossierIds().isEmpty() && !request.dossierIds().contains(dossierEntity.getId())) {
-                    continue;
-                }
+            for (String dossierId : dossierIds) {
 
-                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
+                Dossier dossier = dossierManagementService.getDossierById(dossierId, true, false);
+
+                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossier);
             }
 
             storeZipFile(downloadJob.getStorageId(), archiver);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExportService.java

@@ -29,7 +29,6 @@ import lombok.extern.slf4j.Slf4j;
 public class FileExportService {
 
     ManualChangesExportService manualChangesExportService;
-    ComponentLogService componentLogService;
     ObjectMapper mapper;
     FileManagementStorageService storageService;
 
@@ -41,10 +40,12 @@ public class FileExportService {
         Path fileFolder = folder.resolve(file.getId());
 
         if (!request.excludeAnalysisLogs()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
-                                                                        mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
-            if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
+            if (storageService.entityLogExists(file.getDossierId(), file.getId())) {
+                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                            buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
+                                                                            mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
+            }
+            if (storageService.componentLogExists(file.getDossierId(), file.getId())) {
                 archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
                                                                             buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
                                                                             mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java

@@ -75,4 +75,7 @@ public class ComponentMappingEntity {
     @Builder.Default
     char delimiter = ',';
 
+    @Builder.Default
+    char quoteChar = '"';
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/AnnotationEntityId.java

@@ -12,7 +12,6 @@ import lombok.NoArgsConstructor;
 
 @Embeddable
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class AnnotationEntityId implements Serializable {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java

@@ -17,14 +17,12 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Entity
 @Table(name = "id_removal")
 @Data
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 public class IdRemovalEntity implements IBaseAnnotation {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java

@@ -15,7 +15,7 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
+//@Builder // disabled, cause the ManualChangesExportMapper will not map addToDictionary and addToAllDossiers at all
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
@@ -36,8 +36,6 @@ public class ManualForceRedactionEntity implements IBaseAnnotation {
     private OffsetDateTime softDeletedTime;
     @Column
     private int page;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java

@@ -10,12 +10,10 @@ import jakarta.persistence.Entity;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
@@ -41,8 +39,6 @@ public class ManualLegalBasisChangeEntity implements IBaseAnnotation {
     private OffsetDateTime softDeletedTime;
     @Column
     private int page;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRecategorizationEntity.java

@@ -17,12 +17,10 @@ import jakarta.persistence.Entity;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
@@ -56,8 +54,6 @@ public class ManualRecategorizationEntity implements IBaseAnnotation {
     private String section;
     @Column
     private String value;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java

@@ -22,13 +22,11 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
 @Entity
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Table(name = "manual_redaction")
@@ -51,13 +49,13 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation {
     @Column
     private boolean rectangle;
     @Column
-    private boolean addToDictionary;
+    private boolean addToDictionary; //true, true
     @Column
-    private boolean addToAllDossiers;
+    private boolean addToAllDossiers; //true, false
 
     @Column
     @Deprecated
-    private boolean addToDossierDictionary;
+    private boolean addToDossierDictionary;  //false, true
 
     @Column
     @Enumerated(EnumType.STRING)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java

@@ -19,12 +19,10 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
@@ -63,9 +61,6 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation {
     @Column
     private boolean addToAllDossiers;
 
-    @Column
-    private String basedOnDictAnnotationId;
-
     @ElementCollection
     @Fetch(value = FetchMode.SUBSELECT)
     private Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ApplicationConfigurationEntity.java

@@ -24,15 +24,19 @@ public class ApplicationConfigurationEntity {
     private final String id = ApplicationConfigurationEntity.ID;
 
     @Column
+    @Builder.Default
     private int downloadCleanupDownloadFilesHours = 8;
 
     @Column
+    @Builder.Default
     private int downloadCleanupNotDownloadFilesHours = 72;
 
     @Column
+    @Builder.Default
     private int softDeleteCleanupTime = 96;
 
     @Column
+    @Builder.Default
     private int hardDeleteCleanupRetryTime = 72;
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java

@@ -10,6 +10,7 @@ import org.hibernate.annotations.Fetch;
 import org.hibernate.annotations.FetchMode;
 
 import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
 
@@ -201,6 +202,11 @@ public class FileEntity {
     @Column
     private OffsetDateTime errorTimestamp;
 
+    @Column
+    @Enumerated(EnumType.STRING)
+    private ErrorCode errorCode;
+
+
     @ElementCollection(fetch = FetchType.EAGER)
     private List<FileEntityComponentMappingVersionEntity> componentMappingVersions;
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/SaasMigrationStatusEntity.java

@@ -1,35 +0,0 @@
-package com.iqser.red.service.persistence.management.v1.processor.entity.migration;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
-
-import jakarta.persistence.*;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@Entity
-@Table(name = "saas_migration_status")
-public class SaasMigrationStatusEntity {
-
-    @Id
-    private String fileId;
-
-    @Column
-    private String dossierId;
-
-    @Column
-    @Enumerated(EnumType.STRING)
-    private SaasMigrationStatus status;
-
-    @Column
-    private Integer processingErrorCounter;
-
-    @Column
-    private String errorCause;
-
-}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ManualChangesExportMapper.java

@@ -73,65 +73,76 @@ public interface ManualChangesExportMapper {
 
     @AfterMapping
     default void setFileStatusAndId(ManualLegalBasisChangeExportModel model,
-                                    @MappingTarget ManualLegalBasisChangeEntity.ManualLegalBasisChangeEntityBuilder entity,
-                                    @Context FileEntity file, @Context String userId) {
+                                    @MappingTarget ManualLegalBasisChangeEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
     default void setFileStatusAndId(ManualRecategorizationExportModel model,
-                                    @MappingTarget ManualRecategorizationEntity.ManualRecategorizationEntityBuilder entity,
-                                    @Context FileEntity file, @Context String userId) {
+                                    @MappingTarget ManualRecategorizationEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualAddExportModel model, @MappingTarget ManualRedactionEntryEntity.ManualRedactionEntryEntityBuilder entity, @Context FileEntity file, @Context String userId) {
+    default void setFileStatusAndId(ManualAddExportModel model,
+                                    @MappingTarget ManualRedactionEntryEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualResizeExportModel model, @MappingTarget ManualResizeRedactionEntity.ManualResizeRedactionEntityBuilder entity, @Context FileEntity file, @Context String userId) {
+    default void setFileStatusAndId(ManualResizeExportModel model,
+                                    @MappingTarget ManualResizeRedactionEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualForceExportModel model, @MappingTarget ManualForceRedactionEntity.ManualForceRedactionEntityBuilder entity, @Context FileEntity file, @Context String userId) {
+    default void setFileStatusAndId(ManualForceExportModel model,
+                                    @MappingTarget ManualForceRedactionEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity.IdRemovalEntityBuilder entity, @Context FileEntity file, @Context String userId) {
+    default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity entity, @Context FileEntity file, @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java

@@ -1,170 +0,0 @@
-package com.iqser.red.service.persistence.management.v1.processor.migration;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
-import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
-
-import jakarta.transaction.Transactional;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Service;
-
-@Slf4j
-@Service
-@Transactional
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class SaasAnnotationIdMigrationService {
-
-    ManualRedactionRepository manualRedactionRepository;
-    RemoveRedactionRepository removeRedactionRepository;
-    ForceRedactionRepository forceRedactionRepository;
-    ResizeRedactionRepository resizeRedactionRepository;
-    RecategorizationRepository recategorizationRepository;
-    LegalBasisChangeRepository legalBasisChangeRepository;
-    CommentRepository commentRepository;
-    FileRepository fileRepository;
-
-
-    public int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = manualRedactionRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualRedactionEntryEntity.class);
-            newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class));
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-            newEntry.setId(newAnnotationEntityId);
-
-            manualRedactionRepository.deleteById(oldAnnotationEntityId);
-            manualRedactionRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = removeRedactionRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), IdRemovalEntity.class);
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-            newEntry.setId(newAnnotationEntityId);
-
-            removeRedactionRepository.deleteById(oldAnnotationEntityId);
-            removeRedactionRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = forceRedactionRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualForceRedactionEntity.class);
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-            newEntry.setId(newAnnotationEntityId);
-
-            forceRedactionRepository.deleteById(oldAnnotationEntityId);
-            forceRedactionRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = resizeRedactionRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualResizeRedactionEntity.class);
-            newEntry.setId(newAnnotationEntityId);
-            newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class));
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-
-            resizeRedactionRepository.deleteById(oldAnnotationEntityId);
-            resizeRedactionRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = recategorizationRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualRecategorizationEntity.class);
-            newEntry.setId(newAnnotationEntityId);
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-
-            recategorizationRepository.deleteById(oldAnnotationEntityId);
-            recategorizationRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = legalBasisChangeRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualLegalBasisChangeEntity.class);
-            newEntry.setId(newAnnotationEntityId);
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-
-            legalBasisChangeRepository.deleteById(oldAnnotationEntityId);
-            legalBasisChangeRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateCommentIds(String fileId, String key, String value) {
-
-        if (key.equals(value)) {
-            return 0;
-        }
-        return commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value);
-    }
-
-}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java

@@ -1,83 +0,0 @@
-package com.iqser.red.service.persistence.management.v1.processor.migration;
-
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-
-import org.springframework.stereotype.Service;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
-import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
-
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class SaasMigrationManualChangesUpdateService {
-
-    private final AddRedactionPersistenceService addRedactionPersistenceService;
-
-    private final HashFunction hashFunction = Hashing.murmur3_128();
-
-
-    public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId) {
-
-        var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly());
-        for (var unprocessedManualAdd : unprocessedManualAdds) {
-
-            if (!unprocessedManualAdd.getDictionaryEntryType().equals(DictionaryEntryType.ENTRY)) {
-                continue;
-            }
-
-            if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) {
-                // copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table.
-                // It has no functionality, its only there, such that the UI can show a pending change.
-                ManualRedactionEntryEntity pendingDictAdd = ManualRedactionEntryEntity.builder()
-                        .id(buildSecondaryId(unprocessedManualAdd.getId(), fileId))
-                        .user(unprocessedManualAdd.getUser())
-                        .typeId(unprocessedManualAdd.getTypeId())
-                        .value(unprocessedManualAdd.getValue())
-                        .reason(unprocessedManualAdd.getReason())
-                        .legalBasis(unprocessedManualAdd.getLegalBasis())
-                        .section(unprocessedManualAdd.getSection())
-                        .rectangle(unprocessedManualAdd.isRectangle())
-                        .addToDictionary(unprocessedManualAdd.isAddToDictionary())
-                        .addToAllDossiers(unprocessedManualAdd.isAddToAllDossiers())
-                        .dictionaryEntryType(DictionaryEntryType.ENTRY)
-                        .requestDate(unprocessedManualAdd.getRequestDate())
-                        .positions(new ArrayList<>(unprocessedManualAdd.getPositions())) // copy to new List
-                        .fileStatus(unprocessedManualAdd.getFileStatus())
-                        .textBefore(unprocessedManualAdd.getTextBefore())
-                        .textAfter(unprocessedManualAdd.getTextAfter())
-                        .sourceId(unprocessedManualAdd.getSourceId())
-                        .typeIdsOfModifiedDictionaries(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries())
-                        .build();
-
-                addRedactionPersistenceService.update(pendingDictAdd);
-
-                // change existing dict add to unprocessed manual add. ID must match with prior entry, such that other unprocessed manual changes may be applied to it.
-                unprocessedManualAdd.setAddToDictionary(false);
-                unprocessedManualAdd.setAddToAllDossiers(false);
-                unprocessedManualAdd.setLegalBasis("");
-                unprocessedManualAdd.setTypeIdsOfModifiedDictionaries(Collections.emptySet());
-                unprocessedManualAdd.setDictionaryEntryType(null);
-
-                addRedactionPersistenceService.update(unprocessedManualAdd);
-            }
-        }
-    }
-
-
-    private AnnotationEntityId buildSecondaryId(AnnotationEntityId annotationEntityId, String fileId) {
-
-        return new AnnotationEntityId(hashFunction.hashString(annotationEntityId.getAnnotationId(), StandardCharsets.UTF_8).toString(), fileId);
-    }
-
-}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java

@@ -1,398 +0,0 @@
-package com.iqser.red.service.persistence.management.v1.processor.migration;
-
-import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_REQUEST_QUEUE;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
-import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
-import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
-import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
-import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService;
-import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
-import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
-import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
-import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
-import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
-import com.iqser.red.service.redaction.v1.model.MigrationRequest;
-import com.iqser.red.storage.commons.exception.StorageException;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService;
-import com.knecon.fforesight.databasetenantcommons.providers.events.TenantSyncEvent;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-import com.knecon.fforesight.tenantcommons.TenantProvider;
-import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class SaasMigrationService implements TenantSyncService {
-
-    AutomaticAnalysisJob automaticAnalysisJob;
-    FileStatusPersistenceService fileStatusPersistenceService;
-    SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
-    DossierService dossierService;
-    ManualRedactionProviderService manualRedactionProviderService;
-    TenantProvider tenantProvider;
-    IndexingService indexingService;
-    LayoutParsingRequestFactory layoutParsingRequestFactory;
-    RabbitTemplate rabbitTemplate;
-    FileManagementServiceSettings settings;
-    StorageService storageService;
-    SaasAnnotationIdMigrationService saasAnnotationIdMigrationService;
-    UncompressedFilesMigrationService uncompressedFilesMigrationService;
-    ManualRedactionService manualRedactionService;
-    CommentService commentService;
-    RankDeDuplicationService rankDeDuplicationService;
-    SaasMigrationManualChangesUpdateService saasMigrationManualChangesUpdateService;
-
-
-    @Override
-    public synchronized void syncTenant(TenantSyncEvent tenantSyncEvent) {
-
-        startMigrationForTenant(tenantSyncEvent.getTenantId());
-    }
-
-    // Persistence-Service needs to be scaled to 1.
-
-
-    public void startMigrationForTenant(String tenantId) {
-
-        // TODO migrate rules.
-        automaticAnalysisJob.stopForTenant(tenantId);
-
-        log.info("Starting uncompressed files migration ...");
-        uncompressedFilesMigrationService.migrateUncompressedFiles(tenantId);
-        log.info("Finished uncompressed files migration ...");
-
-        rankDeDuplicationService.deduplicate();
-        int numberOfFiles = 0;
-
-        var files = saasMigrationStatusPersistenceService.findAll();
-
-        for (var file : files) {
-
-            var dossier = dossierService.getDossierById(file.getDossierId());
-
-            if (dossier.getHardDeletedTime() != null) {
-                if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
-                    saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
-                    continue;
-                } else {
-                    fileStatusPersistenceService.hardDelete(file.getFileId(), dossier.getHardDeletedTime());
-                    saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
-                    continue;
-                }
-            }
-            if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
-                saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
-                continue;
-            }
-
-            if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) {
-                log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus());
-                continue;
-            }
-
-            // delete NER_ENTITIES since offsets depend on old document structure.
-            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(file.getDossierId(), file.getFileId(), FileType.NER_ENTITIES));
-
-            var layoutParsingRequest = layoutParsingRequestFactory.build(file.getDossierId(), file.getFileId(), false);
-
-            rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest);
-
-            numberOfFiles++;
-
-        }
-
-        log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId());
-        if (numberOfFiles == 0) {
-            finalizeMigration();
-        }
-    }
-
-
-    public void startMigrationForFile(String dossierId, String fileId) {
-
-        var dossier = dossierService.getDossierById(dossierId);
-
-        if (dossier.getHardDeletedTime() != null) {
-            if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
-                saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
-                return;
-            } else {
-                fileStatusPersistenceService.hardDelete(fileId, dossier.getHardDeletedTime());
-                saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
-                return;
-            }
-        }
-        if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
-            saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
-            return;
-        }
-
-        log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId);
-        saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
-        var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, false);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest);
-    }
-
-
-    public void handleLayoutParsingFinished(String dossierId, String fileId) {
-
-        if (!layoutParsingFilesExist(dossierId, fileId)) {
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Layout parsing files not written!");
-            return;
-        }
-
-        log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
-        saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED);
-
-        if (fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
-            saasMigrationManualChangesUpdateService.convertUnprocessedAddToDictionariesToLocalChanges(fileId);
-        }
-
-        try {
-            indexingService.reindex(dossierId, Set.of(fileId), false);
-
-            String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId();
-
-            rabbitTemplate.convertAndSend(MIGRATION_REQUEST_QUEUE,
-                                          MigrationRequest.builder()
-                                                  .dossierTemplateId(dossierTemplateId)
-                                                  .dossierId(dossierId)
-                                                  .fileId(fileId)
-                                                  .fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED))
-                                                  .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
-                                                  .entitiesWithComments(commentService.getCommentCounts(fileId).keySet())
-                                                  .build());
-        } catch (Exception e) {
-            log.error("Queuing of entityLog migration failed with {}", e.getMessage());
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage()));
-        }
-
-    }
-
-
-    private boolean layoutParsingFilesExist(String dossierId, String fileId) {
-
-        return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) //
-               && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) //
-               && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) //
-               && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION));
-    }
-
-
-    public void handleEntityLogMigrationFinished(String dossierId, String fileId) {
-
-        if (!entityLogMigrationFilesExist(dossierId, fileId)) {
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Migration Files not written!");
-            return;
-        }
-        saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED);
-
-        log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
-        migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(dossierId, fileId);
-    }
-
-
-    private boolean entityLogMigrationFilesExist(String dossierId, String fileId) {
-
-        return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists(
-                TenantContext.getTenantId(),
-                StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS));
-    }
-
-
-    public void handleError(String dossierId, String fileId, String errorCause, String retryExchange) {
-
-        var migrationEntry = saasMigrationStatusPersistenceService.findById(fileId);
-        Integer numErrors = migrationEntry.getProcessingErrorCounter();
-        if (numErrors != null && numErrors <= settings.getMaxErrorRetries()) {
-            saasMigrationStatusPersistenceService.updateErrorCounter(fileId, numErrors + 1, errorCause);
-            rabbitTemplate.convertAndSend(retryExchange, TenantContext.getTenantId(), MigrationRequest.builder().dossierId(dossierId).fileId(fileId).build());
-            log.error("Retrying error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause);
-        } else {
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, errorCause);
-            log.error("Error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause);
-        }
-    }
-
-
-    public void requeueErrorFiles() {
-
-        automaticAnalysisJob.stopForTenant(TenantContext.getTenantId());
-        saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR)
-                .forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId()));
-
-    }
-
-
-    private void migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(String dossierId, String fileId) {
-
-        MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
-        Map<String, String> oldToNewMapping = migratedIds.buildOldToNewMapping();
-        updateAnnotationIds(dossierId, fileId, oldToNewMapping);
-        List<String> forceRedactionIdsToDelete = migratedIds.getForceRedactionIdsToDelete();
-        softDeleteForceRedactions(fileId, forceRedactionIdsToDelete);
-        log.info("Soft-deleted force redactions.");
-        List<ManualRedactionEntry> manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd();
-        int count = addManualRedactionEntries(manualRedactionEntriesToAdd);
-        log.info("Added {} additional manual entries.", count);
-        deleteSectionGridAndNerEntitiesFiles(dossierId, fileId);
-        saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
-
-        log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
-        finalizeMigration(); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
-    }
-
-
-    private void deleteSectionGridAndNerEntitiesFiles(String dossierId, String fileId) {
-
-        try {
-            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
-        } catch (StorageObjectDoesNotExist e) {
-            log.info("No sectiongrid found for {}, {}, ignoring....", dossierId, fileId);
-        }
-
-        try {
-            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
-        } catch (StorageObjectDoesNotExist e) {
-            log.info("No ner entities file found for {}, {}, ignoring....", dossierId, fileId);
-        }
-    }
-
-
-    private void softDeleteForceRedactions(String fileId, List<String> forceRedactionIdsToDelete) {
-
-        manualRedactionService.softDeleteForceRedactions(fileId, forceRedactionIdsToDelete);
-    }
-
-
-    private int addManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
-
-        manualRedactionEntriesToAdd.forEach(add -> {
-            if (add.getSection() != null && add.getSection().length() > 254) {
-                add.setSection(add.getSection().substring(0, 254));
-            }
-        });
-
-        return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true);
-    }
-
-
-    public void revertMigrationForFile(String dossierId, String fileId) {
-
-        log.info("Reverting Migration for dossierId {} and fileId {}", dossierId, fileId);
-        MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
-        Map<String, String> newToOldMapping = migratedIds.buildNewToOldMapping();
-        updateAnnotationIds(dossierId, fileId, newToOldMapping);
-        deleteManualRedactionEntries(migratedIds.getManualRedactionEntriesToAdd());
-        undeleteForceRedactions(fileId, migratedIds.getForceRedactionIdsToDelete());
-        saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
-    }
-
-
-    private void undeleteForceRedactions(String fileId, List<String> forceRedactionIdsToDelete) {
-
-        manualRedactionService.undeleteForceRedactions(fileId, forceRedactionIdsToDelete);
-
-    }
-
-
-    private void deleteManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
-
-        manualRedactionService.deleteManualRedactionEntries(manualRedactionEntriesToAdd);
-    }
-
-
-    private void updateAnnotationIds(String dossierId, String fileId, Map<String, String> idMapping) {
-
-        try {
-            updateAnnotationIds(fileId, idMapping);
-        } catch (Exception e) {
-            String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s",
-                                           TenantContext.getTenantId(),
-                                           dossierId,
-                                           fileId,
-                                           e.getMessage());
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message);
-            log.error(message);
-            throw e;
-        }
-    }
-
-
-    private void finalizeMigration() {
-
-        if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
-//            automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
-            tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build());
-            log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
-        }
-    }
-
-
-    public void updateAnnotationIds(String fileId, Map<String, String> idMapping) {
-
-        AtomicInteger numUpdates = new AtomicInteger(0);
-        AtomicInteger numCommentUpdates = new AtomicInteger(0);
-        idMapping.forEach((key, value) -> {
-            AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key);
-            AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value);
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numCommentUpdates.getAndAdd(saasAnnotationIdMigrationService.updateCommentIds(fileId, key, value));
-        });
-        log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId);
-    }
-
-
-    private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
-
-        return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build();
-    }
-
-
-    private MigratedIds getMigratedIds(String dossierId, String fileId) {
-
-        try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS), MigratedIds.class);
-        } catch (StorageObjectDoesNotExist e) {
-            throw new NotFoundException(String.format("MigratedIds does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId));
-        } catch (StorageException e) {
-            throw new InternalServerErrorException(e.getMessage());
-        }
-    }
-
-}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V10ReduceTextFileSizeMigration.java

@@ -24,7 +24,7 @@ import lombok.extern.slf4j.Slf4j;
 @Setter
 @Service
 @SuppressWarnings("PMD")
-public class ReduceTextFileSizeMigration10 extends Migration {
+public class V10ReduceTextFileSizeMigration extends Migration {
 
     private static final String NAME = "Reduce TEXT filesize migration";
     private static final long VERSION = 10;
@@ -42,7 +42,7 @@ public class ReduceTextFileSizeMigration10 extends Migration {
     private FileManagementStorageService fileManagementStorageService;
 
 
-    public ReduceTextFileSizeMigration10() {
+    public V10ReduceTextFileSizeMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V13MissingFileSizeMigration.java

@@ -17,7 +17,7 @@ import java.util.List;
 @Slf4j
 @Setter
 @Service
-public class MissingFileSizeMigration13 extends Migration {
+public class V13MissingFileSizeMigration extends Migration {
 
     private static final String NAME = "Add missing file sizes";
     private static final long VERSION = 13;
@@ -29,7 +29,7 @@ public class MissingFileSizeMigration13 extends Migration {
     private FileManagementStorageService fileManagementStorageService;
 
 
-    public MissingFileSizeMigration13() {
+    public V13MissingFileSizeMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V14FixDossierDictionaryEntryInRedactionLog.java

@@ -10,13 +10,13 @@ import org.springframework.stereotype.Service;
 @Slf4j
 @Setter
 @Service
-public class FixDossierDictionaryEntryInRedactionLog14 extends Migration {
+public class V14FixDossierDictionaryEntryInRedactionLog extends Migration {
 
     private static final String NAME = "Fix dossier dictionary entries in redactionLog for non dossier dictionaries";
     private static final long VERSION = 14;
 
 
-    public FixDossierDictionaryEntryInRedactionLog14() {
+    public V14FixDossierDictionaryEntryInRedactionLog() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V15ManualRedactionTypeRenameMigration.java

@@ -28,7 +28,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class ManualRedactionTypeRenameMigration15 extends Migration {
+public class V15ManualRedactionTypeRenameMigration extends Migration {
 
     @Autowired
     private FileStatusPersistenceService fileStatusPersistenceService;
@@ -40,7 +40,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration {
     private ManualRedactionService manualRedactionService;
 
 
-    public ManualRedactionTypeRenameMigration15() {
+    public V15ManualRedactionTypeRenameMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V16RankDeDuplicationMigration.java

@@ -12,13 +12,13 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class RankDeDuplicationMigration16 extends Migration {
+public class V16RankDeDuplicationMigration extends Migration {
 
     @Autowired
     private RankDeDuplicationService rankDeDuplicationService;
 
 
-    public RankDeDuplicationMigration16() {
+    public V16RankDeDuplicationMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V17MigrateImportedRedactionsFiles.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
 
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -14,7 +13,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileMan
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 
@@ -24,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class MigrateImportedRedactionsFiles17 extends Migration {
+public class V17MigrateImportedRedactionsFiles extends Migration {
 
     @Autowired
     private FileStatusPersistenceService fileStatusPersistenceService;
@@ -36,7 +34,7 @@ public class MigrateImportedRedactionsFiles17 extends Migration {
     private static final long VERSION = 17;
 
 
-    public MigrateImportedRedactionsFiles17() {
+    public V17MigrateImportedRedactionsFiles() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V18StorageToMongoMigration.java

@@ -11,7 +11,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class StorageToMongoMigration18 extends Migration {
+public class V18StorageToMongoMigration extends Migration {
 
     private final StorageToMongoCopyService storageToMongoCopyService;
 
@@ -19,7 +19,7 @@ public class StorageToMongoMigration18 extends Migration {
     private static final long VERSION = 18;
 
 
-    public StorageToMongoMigration18(StorageToMongoCopyService storageToMongoCopyService) {
+    public V18StorageToMongoMigration(StorageToMongoCopyService storageToMongoCopyService) {
 
         super(NAME, VERSION);
         this.storageToMongoCopyService = storageToMongoCopyService;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V19AddGraphicDictionaryType.java

@@ -16,7 +16,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class AddGraphicDictionaryType19 extends Migration {
+public class V19AddGraphicDictionaryType extends Migration {
 
     @Autowired
     private DossierTemplatePersistenceService dossierTemplatePersistenceService;
@@ -31,7 +31,7 @@ public class AddGraphicDictionaryType19 extends Migration {
     private static final long VERSION = 19;
 
 
-    public AddGraphicDictionaryType19() {
+    public V19AddGraphicDictionaryType() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V20DocumineLayoutRewriteMigration.java

@@ -16,7 +16,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class DocumineLayoutRewriteMigration20 extends Migration  {
+public class V20DocumineLayoutRewriteMigration extends Migration  {
 
     private static final String NAME = "Reanalyse layout for not approved Documine files";
     private static final long VERSION = 20;
@@ -36,7 +36,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration  {
 
 
 
-    public DocumineLayoutRewriteMigration20() {
+    public V20DocumineLayoutRewriteMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V21ComponentOverridesMigration.java

@@ -6,7 +6,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.pdftron.redaction.v1.api.model.ApplicationType;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
 import com.iqser.red.service.persistence.management.v1.processor.migration.StorageToMongoCopyService;
@@ -19,7 +18,7 @@ import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
-public class ComponentOverridesMigration21 extends Migration {
+public class V21ComponentOverridesMigration extends Migration {
 
     private static final String NAME = "Migrate component overrides to mongoDB and create component definitions";
     private static final long VERSION = 21;
@@ -128,7 +127,7 @@ public class ComponentOverridesMigration21 extends Migration {
     StorageToMongoCopyService storageToMongoCopyService;
 
 
-    public ComponentOverridesMigration21() {
+    public V21ComponentOverridesMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V22DocumineHeadlineDetectionMigration.java

@@ -0,0 +1,70 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Setter
+@Service
+public class V22DocumineHeadlineDetectionMigration extends Migration {
+
+    private static final String NAME = "Reanalyse not approved Documine files after new headline detection";
+    private static final long VERSION = 22;
+
+    @Autowired
+    private FileStatusService fileStatusService;
+
+    @Autowired
+    private DossierPersistenceService dossierPersistenceService;
+
+    @Autowired
+    private FileStatusPersistenceService fileStatusPersistenceService;
+
+    @Value("${application.type}")
+    private String applicationType;
+
+
+    public V22DocumineHeadlineDetectionMigration() {
+
+        super(NAME, VERSION);
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Starting migration DocumineHeadlineDetectionMigration22");
+
+        if (!applicationType.equalsIgnoreCase("DocuMine")) {
+            log.info("Skipping DocumineHeadlineDetectionMigration22 as application type is not DocuMine!");
+            return;
+        }
+
+        var dossiers = dossierPersistenceService.findAllDossiers();
+        dossiers.forEach(dossier -> {
+            if (dossier.getHardDeletedTime() == null) {
+                var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId());
+                log.info("Start migration of dossier {}", dossier.getId());
+                files.forEach(file -> {
+                    if (file.getHardDeletedTime() == null && !file.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
+                        log.info("Set full reanalyse for file {}", file.getId());
+                        fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true);
+                        log.info("Finished migration of file {}", file.getId());
+                    }
+                });
+                log.info("Finished migration of dossier {}", dossier.getId());
+            }
+        });
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V23ManualChangesReorderingMigration.java

@@ -0,0 +1,90 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import java.time.OffsetDateTime;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.projections.EntryWithManualChangesProjection;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class V23ManualChangesReorderingMigration extends Migration {
+
+    private static final String NAME = "Migration for reordering mixed up manual changes";
+    private static final long VERSION = 23;
+    private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
+    private final AddRedactionPersistenceService addRedactionPersistenceService;
+    private final FileStatusService fileStatusService;
+
+
+    public V23ManualChangesReorderingMigration(EntityLogEntryDocumentRepository entityLogEntryDocumentRepository,
+                                               AddRedactionPersistenceService addRedactionPersistenceService,
+                                               FileStatusService fileStatusService) {
+
+        super(NAME, VERSION);
+        this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;
+        this.addRedactionPersistenceService = addRedactionPersistenceService;
+        this.fileStatusService = fileStatusService;
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Searching for entityLogEntries of type ENTITY with state APPLIED and no legalbasis");
+        List<EntryWithManualChangesProjection> entriesWithManualChanges = entityLogEntryDocumentRepository.findAppliedEntitiesWhereLegalBasisEmpty();
+        log.info("Found {} applied entries with no legal basis", entriesWithManualChanges.size());
+        int numberOfChanged = 0;
+        Set<FileAndDossier> affectedFiles = new HashSet<>();
+        for (EntryWithManualChangesProjection entry : entriesWithManualChanges) {
+            if (entry.getManualChanges().isEmpty()) {
+                continue;
+            }
+            Optional<ManualChange> optionalAdd = entry.getManualChanges()
+                    .stream()
+                    .filter(mc -> mc.getManualRedactionType().equals(ManualRedactionType.ADD))
+                    .findFirst();
+            if (optionalAdd.isEmpty()) {
+                continue;
+            }
+            ManualChange add = optionalAdd.get();
+            ManualChange first = entry.getManualChanges()
+                    .get(0);
+            if (add.equals(first)) {
+                continue;
+            }
+            OffsetDateTime firstTimestamp = first.getRequestedDate().minusSeconds(1);
+            String dossierId = entry.getEntityLogId().split("/")[0];
+            String fileId = entry.getEntityLogId().split("/")[1];
+            int i = addRedactionPersistenceService.updateRequestDate(fileId, entry.getEntryId(), firstTimestamp);
+            if (i != 0) {
+                affectedFiles.add(new FileAndDossier(fileId, dossierId));
+                numberOfChanged += i;
+            }
+        }
+        log.info("Updated request date of {} manual add redactions in {} files.", numberOfChanged, affectedFiles.size());
+        log.info("Reanalyzing affected files");
+        for (FileAndDossier affectedFile : affectedFiles) {
+            fileStatusService.setStatusReprocess(affectedFile.dossierId(), affectedFile.fileId(), false);
+        }
+
+    }
+
+
+    private record FileAndDossier(String fileId, String dossierId) {
+
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V24StorageToMongoMigration.java

@@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 @Setter
 @Service
 
-public class StorageToMongoMigration24 extends Migration {
+public class V24StorageToMongoMigration extends Migration {
 
     private final StorageToMongoCopyService storageToMongoCopyService;
     private final MongoTemplate mongoTemplate;
@@ -29,7 +29,7 @@ public class StorageToMongoMigration24 extends Migration {
     private static final long VERSION = 24;
 
 
-    public StorageToMongoMigration24(StorageToMongoCopyService storageToMongoCopyService, MongoTemplate mongoTemplate) {
+    public V24StorageToMongoMigration(StorageToMongoCopyService storageToMongoCopyService, MongoTemplate mongoTemplate) {
 
         super(NAME, VERSION);
         this.storageToMongoCopyService = storageToMongoCopyService;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V25AddDateFormatsToTemplatesMigration.java

@@ -1,19 +1,14 @@
 package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
 
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.DefaultDateFormatsProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 
@@ -24,7 +19,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class AddDateFormatsToTemplatesMigration25 extends Migration {
+public class V25AddDateFormatsToTemplatesMigration extends Migration {
 
     private static final String NAME = "Migration for adding date formats files to dossier templates";
     private static final long VERSION = 25;
@@ -33,12 +28,14 @@ public class AddDateFormatsToTemplatesMigration25 extends Migration {
     DateFormatsPersistenceService dateFormatsPersistenceService;
     @Autowired
     DossierTemplatePersistenceService dossierTemplatePersistenceService;
-
+    @Autowired
+    DefaultDateFormatsProvider defaultDateFormatsProvider;
 
     @Value("${application.type}")
     private String applicationType;
 
-    public AddDateFormatsToTemplatesMigration25() {
+
+    public V25AddDateFormatsToTemplatesMigration() {
 
         super(NAME, VERSION);
     }
@@ -48,19 +45,15 @@ public class AddDateFormatsToTemplatesMigration25 extends Migration {
     @SneakyThrows
     protected void migrate() {
 
-        if(!applicationType.equalsIgnoreCase("DocuMine")){
+        if (!applicationType.equalsIgnoreCase("DocuMine")) {
             log.info("Skipping AddDateFormatsToTemplatesMigration25 as application type is not DocuMine!!!");
             return;
         }
 
         List<DossierTemplateEntity> allDossierTemplates = dossierTemplatePersistenceService.getAllDossierTemplates();
         allDossierTemplates.forEach(dt -> {
-            Resource resource = new ClassPathResource("files/dateFormats.txt");
-
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) {
-                String dateFormats = reader.lines().collect(Collectors.joining(System.lineSeparator()));
-                dateFormatsPersistenceService.setDateFormats(dateFormats, dt.getId(), -1);
-
+            try {
+                dateFormatsPersistenceService.setDateFormats(defaultDateFormatsProvider.getDateFormats(), dt.getId(), -1);
             } catch (Exception e) {
                 log.info("Could not update dossier template {}, error: {}", dt.getId(), e.getMessage());
             }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V26RuleFileUpdateMigration.java

@@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class RuleFileUpdateMigration26 extends Migration {
+public class V26RuleFileUpdateMigration extends Migration {
 
     private final DossierTemplateRepository dossierTemplateRepository;
     private final RulesPersistenceService rulesPersistenceService;
@@ -29,7 +29,7 @@ public class RuleFileUpdateMigration26 extends Migration {
     private static final long VERSION = 26;
 
 
-    public RuleFileUpdateMigration26(DossierTemplateRepository dossierTemplateRepository, RulesPersistenceService rulesPersistenceService) {
+    public V26RuleFileUpdateMigration(DossierTemplateRepository dossierTemplateRepository, RulesPersistenceService rulesPersistenceService) {
 
         super(NAME, VERSION);
         this.dossierTemplateRepository = dossierTemplateRepository;
@@ -62,7 +62,7 @@ public class RuleFileUpdateMigration26 extends Migration {
                     TenantContext.setTenantId(tenantId);
                     String updatedRules = ruleSet.getValue()
                             .replaceAll("import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;",
-                                        "import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;");
+                                        "import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;");
                     rulesPersistenceService.setRules(updatedRules, ruleSet.getDossierTemplateId(), RuleFileType.ENTITY);
                 });
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V27QueueRenameMigration.java

@@ -14,7 +14,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class QueueRenameMigration27 extends Migration {
+public class V27QueueRenameMigration extends Migration {
 
     private final AmqpAdmin amqpAdmin;
 
@@ -86,7 +86,7 @@ public class QueueRenameMigration27 extends Migration {
                                                            "visual_layout_parsing_service_response_queue");
 
 
-    public QueueRenameMigration27(AmqpAdmin amqpAdmin) {
+    public V27QueueRenameMigration(AmqpAdmin amqpAdmin) {
 
         super(NAME, VERSION);
         this.amqpAdmin = amqpAdmin;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V28AddTechnicalNameToJustifications.java

@@ -12,15 +12,15 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class AddTechnicalNameToJustifications22 extends Migration {
+public class V28AddTechnicalNameToJustifications extends Migration {
 
     private static final String NAME = "Migration to add a technical name to justifications";
-    private static final long VERSION = 22;
+    private static final long VERSION = 28;
     @Autowired
     private LegalBasisMigrationService legalBasisMigrationService;
 
 
-    public AddTechnicalNameToJustifications22() {
+    public V28AddTechnicalNameToJustifications() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/DossierIdFilterable.java

@@ -0,0 +1,6 @@
+package com.iqser.red.service.persistence.management.v1.processor.model;
+
+public interface DossierIdFilterable {
+
+    String getDossierId();
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ApprovalVerificationService.java

@@ -53,7 +53,7 @@ public class ApprovalVerificationService {
                     var legalBasisEntity = legalBasisMappings.stream()
                             .filter(mapping -> mapping.getReason().equals(entry.getLegalBasis()))
                             .findFirst();
-                    if (legalBasisEntity.isEmpty() || StringUtils.isEmpty(legalBasisEntity.get().getTechnicalName())) {
+                    if (legalBasisEntity.isEmpty() || StringUtils.isEmpty(legalBasisEntity.get().getReason())) {
                         addWarning(entry, WarningType.UNMAPPED_JUSTIFICATION, approveResponse);
                     }
                 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java

@@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.management.v1.processor.mapper.Componen
 import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping;
 import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
 import com.opencsv.CSVParserBuilder;
 import com.opencsv.CSVReader;
@@ -62,16 +63,23 @@ public class ComponentMappingService {
 
 
     @SneakyThrows
-    public ComponentMappingMetadata update(String dossierTemplateId, String mappingId, String name, String encoding, char delimiter, File mappingFile, String fileName) {
+    public ComponentMappingMetadata update(String dossierTemplateId,
+                                           String mappingId,
+                                           String name,
+                                           String encoding,
+                                           char delimiter,
+                                           File mappingFile,
+                                           String fileName,
+                                           char quoteChar) {
 
         ComponentMappingEntity entity = componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId);
 
-        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName);
+        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName, quoteChar);
     }
 
 
     @SneakyThrows
-    public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile) {
+    public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile, char quoteChar) {
 
         if (componentMappingPersistenceService.existsByNameAndDossierTemplateId(name, dossierTemplateId)) {
             throw new BadRequestException("A mapping with this name already exists in the dossier template!");
@@ -86,20 +94,27 @@ public class ComponentMappingService {
                 .fileName(fileName)
                 .build();
 
-        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName);
+        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName, quoteChar);
 
     }
 
 
     @SneakyThrows
-    private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity, String name, String encoding, char delimiter, File mappingFile, String fileName) {
+    private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity,
+                                                    String name,
+                                                    String encoding,
+                                                    char delimiter,
+                                                    File mappingFile,
+                                                    String fileName,
+                                                    char quoteChar) {
 
-        Charset charset = resolveCharset(encoding);
+        Charset charset = StringEncodingUtils.resolveCharset(encoding);
 
-        CsvStats stats = sortCSVFile(delimiter, mappingFile, charset);
+        CsvStats stats = sortCSVFile(delimiter, mappingFile, charset, quoteChar);
 
         entity.setName(name);
         entity.setDelimiter(delimiter);
+        entity.setQuoteChar(quoteChar);
         entity.setEncoding(encoding);
         entity.setNumberOfLines(stats.numberOfLines());
         entity.setColumnLabels(stats.columnLabels());
@@ -112,21 +127,7 @@ public class ComponentMappingService {
     }
 
 
-    private static Charset resolveCharset(String encoding) {
-
-        try {
-            return Charset.forName(encoding);
-        } catch (IllegalCharsetNameException e) {
-            throw new BadRequestException("Invalid character encoding: " + encoding);
-        } catch (UnsupportedCharsetException e) {
-            throw new BadRequestException("Unsupported character encoding: " + encoding);
-        } catch (IllegalArgumentException e) {
-            throw new BadRequestException("Encoding can't be null.");
-        }
-    }
-
-
-    private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws BadRequestException, IOException {
+    private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset, char quoteChar) throws BadRequestException, IOException {
 
         Path tempFile = Files.createTempFile("mapping", ".tmp");
 
@@ -135,11 +136,8 @@ public class ComponentMappingService {
         String[] columnLabels;
         int numberOfLines = 0;
         try (Reader fileReader = new FileReader(tempFile.toFile(), charset);//
-             CSVReader reader = buildReader(fileReader, delimiter);//
-             CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset), delimiter,
-                                              CSVWriter.NO_QUOTE_CHARACTER,
-                                              CSVWriter.DEFAULT_ESCAPE_CHARACTER,
-                                              CSVWriter.DEFAULT_LINE_END)) {
+             CSVReader reader = buildReader(fileReader, delimiter, quoteChar);//
+             CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset), delimiter, quoteChar, '\\', CSVWriter.DEFAULT_LINE_END)) {
 
             List<String[]> rows = reader.readAll();
 
@@ -180,9 +178,9 @@ public class ComponentMappingService {
     }
 
 
-    private static CSVReader buildReader(Reader reader, char delimiter) throws IOException {
+    private static CSVReader buildReader(Reader reader, char delimiter, char quoteChar) throws IOException {
 
-        return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build();
+        return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).withQuoteChar(quoteChar).build()).build();
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DefaultDateFormatsProvider.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.persistence.management.v1.processor.service;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.stream.Collectors;
+
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.stereotype.Service;
+
+import lombok.SneakyThrows;
+
+@Service
+public class DefaultDateFormatsProvider {
+
+    private String dateFormats;
+
+
+    @SneakyThrows
+    public String getDateFormats() {
+
+        if (dateFormats == null) {
+            Resource resource = new ClassPathResource("files/dateFormats.txt");
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) {
+                dateFormats = reader.lines()
+                        .collect(Collectors.joining(System.lineSeparator()));
+            }
+        }
+
+        return dateFormats;
+    }
+
+}
+

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java

@@ -189,8 +189,6 @@ public class DictionaryService {
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DOSSIER_DICTIONARY_TYPE + "')")
     public void updateDossierType(String type, String dossierTemplateId, UpdateTypeValue typeValue, String dossierId, TypeEntity typeEntity) {
 
-        accessControlService.verifyUserIsDossierOwner(dossierId);
-
         if (typeEntity.isDossierDictionaryOnly()) {
             dictionaryManagementService.updateTypeValue(toTypeId(type, dossierTemplateId, dossierId),
                                                         Type.builder()
@@ -264,7 +262,7 @@ public class DictionaryService {
     @PreAuthorize("hasAuthority('" + DELETE_DOSSIER_DICTIONARY_TYPE + "')")
     public void deleteDossierType(String type, String dossierTemplateId, String dossierId) {
 
-        accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
+        accessControlService.checkAccessPermissionsToDossier(dossierId);
         accessControlService.verifyUserIsMemberOrApprover(dossierId);
         deleteType(toTypeId(type, dossierTemplateId, dossierId));
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java

@@ -23,11 +23,12 @@ public class DossierIdFileIdRequestValidator {
     DossierManagementService dossierManagementService;
     FileStatusManagementService fileStatusManagementService;
 
+
     @Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request")
     public void validateRequestOrThrow404(String dossierTemplateId, Set<String> dossierIds, Set<String> fileIds) {
 
         if (!dossierIds.isEmpty()) {
-            Set<String> availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
+            Set<String> availableDossierIds = new HashSet<>(dossierManagementService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, dossierIds));
             Set<String> nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds);
             if (!nonAvailableDossiers.isEmpty()) {
                 throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", String.join(", ", nonAvailableDossiers), dossierTemplateId));
@@ -35,9 +36,19 @@ public class DossierIdFileIdRequestValidator {
         }
 
         if (!fileIds.isEmpty()) {
-            Set<String> availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplateId)
+            Set<FileModel> availableFiles = fileStatusManagementService.findAllByIds(fileIds)
                     .stream()
                     .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
+                    .collect(Collectors.toSet());
+
+            Set<String> mentionedDossierIds = availableFiles.stream()
+                    .map(FileModel::getDossierId)
+                    .collect(Collectors.toSet());
+
+            Set<String> availableDossierIds = new HashSet<>(dossierManagementService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, mentionedDossierIds));
+
+            Set<String> availableFileIds = availableFiles.stream()
+                    .filter(file -> availableDossierIds.contains(file.getDossierId()))
                     .map(FileModel::getId)
                     .collect(Collectors.toSet());
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.utils.DossierMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierInformation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
@@ -120,9 +121,15 @@ public class DossierManagementService {
     }
 
     @Transactional
-    public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
 
-        return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
+        return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
+    @Transactional
+    public List<String> findAllDossierIdsInDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
+
+        return dossierService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, dossierIds);
 
     }
 
@@ -269,4 +276,16 @@ public class DossierManagementService {
 
     }
 
+
+    public DossierChangeResponseV2 changesSinceV2(JSONPrimitive<OffsetDateTime> since) {
+        return dossierService.changesSinceV2(since.getValue());
+    }
+
+
+    @Transactional
+    public List<Dossier> getDossiersByIds(Set<String> viewableDossierIds) {
+
+        return getConvertedAllDossiers(dossierService.getAllDossiers(viewableDossierIds), true,true);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
 import java.time.OffsetDateTime;
+import java.util.Collection;
 import java.util.List;
 import java.util.Set;
 
@@ -14,6 +15,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierChange;
@@ -147,7 +149,7 @@ public class DossierService {
     }
 
 
-    public List<DossierEntity> getAllDossiers(List<String> dossierIds) {
+    public List<DossierEntity> getAllDossiers(Collection<String> dossierIds) {
 
         return dossierPersistenceService.findAllDossiers(dossierIds);
     }
@@ -159,12 +161,17 @@ public class DossierService {
     }
 
 
-    public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> findAllDossierIdsInDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
 
-        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId);
+        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId, dossierIds);
     }
 
 
+    public List<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
+
+        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
     public Set<DossierChange> changesSince(OffsetDateTime since) {
 
         return dossierPersistenceService.hasChangesSince(since);
@@ -182,4 +189,9 @@ public class DossierService {
         dossierPersistenceService.unarchiveDossier(dossierId);
     }
 
+
+    public DossierChangeResponseV2 changesSinceV2(OffsetDateTime value) {
+        return dossierPersistenceService.hasChangesSinceV2(value);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java

@@ -156,7 +156,7 @@ public class DossierTemplateCloneService {
 
     private void cloneComponents(String dossierTemplate, String clonedDossierTemplateId) {
 
-        List<ComponentDefinitionEntity> componentDefinitionEntities = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplate);
+        List<ComponentDefinitionEntity> componentDefinitionEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierTemplate);
 
         for (ComponentDefinitionEntity componentDefinitionEntity : componentDefinitionEntities) {
             ComponentDefinitionAddRequest componentDefinitionAddRequest = ComponentDefinitionAddRequest.builder()
@@ -216,7 +216,8 @@ public class DossierTemplateCloneService {
                                            componentMapping.metaData().getFileName(),
                                            componentMapping.metaData().getDelimiter(),
                                            componentMapping.metaData().getEncoding(),
-                                           componentMapping.file());
+                                           componentMapping.file(),
+                                           componentMapping.metaData().getQuoteChar());
         }
         FileSystemUtils.deleteRecursively(dir);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java

@@ -103,13 +103,17 @@ public class EntityLogMergeService {
                                        int analysisNumber,
                                        Map<String, List<BaseAnnotation>> allManualChanges) {
 
-        Map<String, String> trackLocalChangesBasedOnDictEntriesMap = new HashMap<>();
         List<EntityLogEntry> mergedEntityLogEntries = new LinkedList<>(entityLogEntries);
         Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber)//
                 .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
         mergedEntityLogEntries.addAll(addedLocalManualEntries.values());
         buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(mergedEntityLogEntries::add);
 
+        Map<String, String> trackLocalChangesBasedOnDictEntriesMap = unprocessedManualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(ManualRedactionEntry::isLocal)
+                .filter(entry -> entry.getSourceId() != null && !entry.getSourceId().isEmpty())
+                .collect(Collectors.toMap(ManualRedactionEntry::getAnnotationId, ManualRedactionEntry::getSourceId));
         processEntityLogEntries(dossier, mergedEntityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges, trackLocalChangesBasedOnDictEntriesMap);
 
         adjustEntityLogEntriesAfterLocalChangesBasedOnDict(entityLogEntries, trackLocalChangesBasedOnDictEntriesMap, analysisNumber);
@@ -121,7 +125,7 @@ public class EntityLogMergeService {
                                                                     Map<String, String> trackLocalChangesBasedOnDictEntriesMap,
                                                                     int analysisNumber) {
 
-        var dictEntryIdsToUpdate = trackLocalChangesBasedOnDictEntriesMap.values();
+        Set<String> dictEntryIdsToUpdate = new HashSet<>(trackLocalChangesBasedOnDictEntriesMap.values());
         entityLogEntries.stream()
                 .filter(entityLogEntry -> dictEntryIdsToUpdate.contains(entityLogEntry.getId()))
                 .forEach(entityLogEntry -> {
@@ -268,26 +272,14 @@ public class EntityLogMergeService {
                         return null;
                     } else if (localChange instanceof ManualResizeRedaction manualResizeRedaction) {
                         mergeResizeRedaction(manualResizeRedaction, entityLogEntry, analysisNumber);
-                        if (manualResizeRedaction.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualResizeRedaction.getAnnotationId(), manualResizeRedaction.getBasedOnDictAnnotationId());
-                        }
 
                         return null;
                     } else if (localChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
                         mergeLegalBasisChange(manualLegalBasisChange, entityLogEntry, analysisNumber);
-                        if (manualLegalBasisChange.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualLegalBasisChange.getAnnotationId(), manualLegalBasisChange.getBasedOnDictAnnotationId());
-                        }
                         return null;
                     } else if (localChange instanceof ManualRecategorization manualRecategorization) {
-                        if (manualRecategorization.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualRecategorization.getAnnotationId(), manualRecategorization.getBasedOnDictAnnotationId());
-                        }
                         return mergeRecategorization(manualRecategorization, entityLogEntry, dossier, analysisNumber);
                     } else if (localChange instanceof ManualForceRedaction manualForceRedaction) {
-                        if (manualForceRedaction.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualForceRedaction.getAnnotationId(), manualForceRedaction.getBasedOnDictAnnotationId());
-                        }
                         mergeForceRedaction(manualForceRedaction, entityLogEntry, analysisNumber);
                         return null;
                     } else {
@@ -393,11 +385,10 @@ public class EntityLogMergeService {
 
         entityLogEntry.setState(EntryState.IGNORED);
         //special case, only for add local and remove only
-        if (entityLogEntry.getEngines().equals(Set.of(Engine.MANUAL))) {
+        if (entityLogEntry.getEngines().contains(Engine.MANUAL)) {
             entityLogEntry.setState(EntryState.REMOVED);
             change.setType(ChangeType.REMOVED);
         }
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(idRemoval, 0));
 
         changes.add(change);
@@ -435,7 +426,6 @@ public class EntityLogMergeService {
         entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
         entityLogEntry.setPositions(newPositions);
         entityLogEntry.setValue(manualResizeRedaction.getValue());
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualResizeRedaction, 0));
     }
 
@@ -464,7 +454,6 @@ public class EntityLogMergeService {
         entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
         entityLogEntry.setSection(manualLegalBasisChange.getSection());
         entityLogEntry.setValue(manualLegalBasisChange.getValue());
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualLegalBasisChange, 0));
     }
 
@@ -486,8 +475,6 @@ public class EntityLogMergeService {
             return pendingEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
         }
 
-        entityLogEntry.getEngines().add(Engine.MANUAL);
-
         if (recategorization.getType() != null && !recategorization.getType().equals(entityLogEntry.getType())) {
             boolean isHint = isHint(recategorization.getType(), dossier);
             entityLogEntry.setType(recategorization.getType());
@@ -549,10 +536,6 @@ public class EntityLogMergeService {
                                            PropertyChange.builder().property("state").oldValue(oldState.name()).newValue(newState.name()).build()));
         entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
         entityLogEntry.setState(newState);
-        entityLogEntry.getEngines().add(Engine.MANUAL);
-        if (forceRedaction.getBasedOnDictAnnotationId() != null) {
-            entityLogEntry.getEngines().add(Engine.DICTIONARY);
-        }
         addChanges(entityLogEntry, changes);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(forceRedaction, 0));
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
+import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ASCII_ENCODING;
+import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ISO_ENCODING;
 import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeTypeFormats.FILE_ATTRIBUTE_TYPE_DATE_FORMAT;
 import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeTypeFormats.FILE_ATTRIBUTE_TYPE_NUMBER_REGEX;
 
@@ -32,6 +34,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Confl
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ImportCsvRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ImportCsvResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
@@ -59,10 +62,6 @@ public class FileAttributesManagementService {
     private final DossierPersistenceService dossierPersistenceService;
     private final IndexingService indexingService;
 
-    public static String UTF_ENCODING = "UTF-8";
-    public static String ASCII_ENCODING = "ASCII";
-    public static String ISO_ENCODING = "ISO";
-
 
     @Transactional
     public ImportCsvResponse importCsv(String dossierId, ImportCsvRequest importCsvRequest) {
@@ -144,7 +143,7 @@ public class FileAttributesManagementService {
             throw new IllegalArgumentException("Delimiter must be a single character.");
         }
         char delimiterChar = delimiter.charAt(0);
-        Charset charset = Charset.forName(encoding);
+        Charset charset = StringEncodingUtils.resolveCharset(encoding);
 
         try (CSVReader csvReader = new CSVReaderBuilder(new InputStreamReader(new ByteArrayInputStream(csvFileBytes), charset)).withCSVParser(new CSVParserBuilder().withSeparator(
                 delimiterChar).build()).build()) {
@@ -214,7 +213,7 @@ public class FileAttributesManagementService {
         if (ASCII_ENCODING.equalsIgnoreCase(encoding) || StandardCharsets.US_ASCII.name().equalsIgnoreCase(encoding)) {
             return StandardCharsets.US_ASCII;
         }
-        // accept both "ISO" (non-unique name) and the actual name "US-ASCII" of the charset
+        // accept only name "ISO_8859_1" of the charset
         if (ISO_ENCODING.equalsIgnoreCase(encoding) || StandardCharsets.ISO_8859_1.name().equalsIgnoreCase(encoding)) {
             return StandardCharsets.ISO_8859_1;
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java

@@ -20,7 +20,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
 import com.iqser.red.service.search.v1.model.IndexMessageType;
 
-import groovy.transform.Field;
 import jakarta.transaction.Transactional;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -44,7 +43,6 @@ public class FileDeletionService {
     FileStatusPersistenceService fileStatusPersistenceService;
     FileManagementStorageService fileManagementStorageService;
     IndexingService indexingService;
-    ComponentLogService componentLogService;
     ComponentLogMongoService componentLogMongoService;
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java

@@ -170,6 +170,11 @@ public class FileManagementStorageService {
         return entityLogMongoService.entityLogDocumentExists(dossierId, fileId);
     }
 
+    public boolean componentLogExists(String dossierId, String fileId) {
+
+        return componentLogMongoService.componentLogDocumentExists(dossierId, fileId);
+    }
+
 
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 
@@ -246,6 +251,8 @@ public class FileManagementStorageService {
 
     public void deleteAllObjects(String dossierId, String fileId) {
 
+        deleteObject(dossierId, fileId, FileType.VIEWER_DOCUMENT);
+
         deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
         deleteEntityLog(dossierId, fileId);
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusManagementService.java

@@ -23,7 +23,6 @@ public class FileStatusManagementService {
 
     private final FileStatusService fileStatusService;
     private final ExcludeFromAnalysisService excludeFromAnalysis;
-    private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
     private final IndexingService indexingService;
 
 
@@ -38,11 +37,13 @@ public class FileStatusManagementService {
         return fileStatusService.getAllFiles();
     }
 
+
     public List<FileModel> getAllDossierTemplateStatus(String dossierTemplateId) {
 
         return fileStatusService.getDossierTemplateStatus(dossierTemplateId);
     }
 
+
     public List<FileModel> getDossierStatus(String dossierId) {
 
         return fileStatusService.getDossierStatus(dossierId)
@@ -51,6 +52,16 @@ public class FileStatusManagementService {
                 .collect(Collectors.toList());
     }
 
+    public List<FileModel> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) {
+
+        return fileStatusService.findAllDossierIdAndIds(dossierId,fileIds);
+    }
+
+
+    public List<String> getDossierStatusIds(String dossierId, boolean includeDeleted) {
+
+        return fileStatusService.getDossierStatusIds(dossierId, includeDeleted);
+    }
 
 
     public List<FileModel> getAllDossierStatus(String dossierId) {
@@ -194,4 +205,10 @@ public class FileStatusManagementService {
         fileStatusService.setExcludedPages(fileId, excludedPages);
     }
 
+
+    public List<FileModel> findAllByIds(Set<String> fileIds) {
+
+        return fileStatusService.findAllByIds(fileIds);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java

@@ -123,7 +123,7 @@ public class FileStatusProcessingUpdateService {
         } else {
             fileStatusService.setStatusOcrProcessing(fileId,
                                                      fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
-            fileStatusService.addToOcrQueue(dossierId, fileId, 2);
+            fileStatusService.addToOcrQueue(dossierId, fileId, 2, false);
         }
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java

@@ -5,13 +5,12 @@ import static com.iqser.red.service.persistence.management.v1.processor.configur
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.BiFunction;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Value;
@@ -24,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.configuration.M
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ComponentDefinitionEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@@ -76,6 +76,7 @@ import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverte
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.knecon.fforesight.llm.service.LlmNerMessage;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
 import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
@@ -144,6 +145,7 @@ public class FileStatusService {
         return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
     }
 
+
     public List<DossierStatsFileProjection> getDossierStatsFiles(String dossierId) {
 
         return fileStatusPersistenceService.getFilesForDossierStats(dossierId);
@@ -315,7 +317,7 @@ public class FileStatusService {
             }
 
             log.info("Add file: {} from dossier {} to OCR queue", fileId, dossierId);
-            setStatusOcrQueued(dossierId, fileId);
+            setStatusOcrQueued(dossierId, fileId, false);
             sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
             return;
         }
@@ -544,7 +546,7 @@ public class FileStatusService {
     }
 
 
-    public void setStatusOcrQueued(String dossierId, String fileId) {
+    public void setStatusOcrQueued(String dossierId, String fileId, boolean allPages) {
 
         FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
 
@@ -556,7 +558,7 @@ public class FileStatusService {
         updateOCRStartTime(fileId);
         fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING_QUEUED);
         websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.OCR_PROCESSING, fileStatus.getNumberOfAnalyses() + 1);
-        addToOcrQueue(dossierId, fileId, 2);
+        addToOcrQueue(dossierId, fileId, 2, allPages);
     }
 
 
@@ -715,16 +717,32 @@ public class FileStatusService {
     }
 
 
-    public void addToOcrQueue(String dossierId, String fileId, int priority) {
+    public void addToOcrQueue(String dossierId, String fileId, int priority, boolean allPages) {
 
         var removeWatermark = dossierTemplatePersistenceService.getDossierTemplate(dossierPersistenceService.getDossierTemplateId(dossierId)).isRemoveWatermark();
-        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_REQUEST_EXCHANGE,
-                                      TenantContext.getTenantId(),
-                                      new DocumentRequest(dossierId, fileId, removeWatermark),
-                                      message -> {
-                                          message.getMessageProperties().setPriority(priority);
-                                          return message;
-                                      });
+        Set<AzureOcrFeature> features = new HashSet<>();
+        if (removeWatermark) {
+            features.add(AzureOcrFeature.REMOVE_WATERMARKS);
+        }
+        if (allPages) {
+            features.add(AzureOcrFeature.ALL_PAGES);
+        }
+        if (applicationType.equals("DocuMine")) {
+            features.add(AzureOcrFeature.ROTATION_CORRECTION);
+            features.add(AzureOcrFeature.FONT_STYLE_DETECTION);
+        }
+
+        var request = DocumentRequest.builder()
+                .dossierId(dossierId)
+                .fileId(fileId)
+                .removeWatermark(features.contains(AzureOcrFeature.REMOVE_WATERMARKS)) // needed for legacy OCR-services
+                .features(features)
+                .build();
+
+        rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_REQUEST_EXCHANGE, TenantContext.getTenantId(), request, message -> {
+            message.getMessageProperties().setPriority(priority);
+            return message;
+        });
     }
 
 
@@ -1066,4 +1084,31 @@ public class FileStatusService {
         addSearchTermOccurrencesAnalysisRequestToAnalysisQueue(fileStatus, bulkLocalRequest);
     }
 
+
+    public List<FileModel> findAllByIds(Set<String> fileIds) {
+
+        return fileStatusPersistenceService.findAllByIds(fileIds)
+                .stream()
+                .map(entity -> MagicConverter.convert(entity, FileModel.class, new FileModelMapper()))
+                .collect(Collectors.toList());
+    }
+
+
+    public List<String> getDossierStatusIds(String dossierId, boolean includeDeleted) {
+
+        return fileStatusPersistenceService.findAllByDossierId(dossierId, includeDeleted);
+    }
+
+
+    @Transactional
+    public List<FileModel> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) {
+
+        var fileModels = fileStatusPersistenceService.findAllDossierIdAndIds(dossierId, fileIds)
+                .stream()
+                .map(entity -> MagicConverter.convert(entity, FileModel.class, new FileModelMapper()))
+                .toList();
+
+        return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModels);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FilterByPermissionsService.java

@@ -7,6 +7,7 @@ import org.springframework.security.access.prepost.PostAuthorize;
 import org.springframework.security.access.prepost.PreFilter;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.IHavingDossierId;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 
 /*
@@ -17,6 +18,20 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 @Service
 public class FilterByPermissionsService {
 
+    @PreFilter("hasPermission(filterObject.getDossierId(), 'Dossier', 'VIEW_OBJECT')")
+    public <T extends IHavingDossierId> List<T> onlyViewableHavingDossierId(List<T> items) {
+
+        return items;
+
+    }
+
+    @PreFilter("hasPermission(filterObject, 'Dossier', 'VIEW_OBJECT')")
+    public Set<String> onlyViewableDossierIds(Set<String> dossierIds) {
+
+        return dossierIds;
+
+    }
+
     @PreFilter("hasPermission(filterObject, 'Dossier', 'VIEW_OBJECT')")
     public List<String> onlyViewableDossierIds(List<String> dossierIds) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java

@@ -25,6 +25,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 
@@ -60,8 +61,9 @@ public class ReanalysisRequiredStatusService {
         Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>();
         Map<String, Long> dossierVersionMap = new HashMap<>();
         Map<String, DossierEntity> dossierMap = new HashMap<>();
+        Map<String, Map<String, Integer>> componentMappingVersionMap = new HashMap<>();
         fileModels.forEach(entry -> {
-            var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
+            var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap, componentMappingVersionMap);
             entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
             entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
             entry.setComponentReanalysisRequired(analysisRequiredResult.isComponentReanalysisRequired());
@@ -75,7 +77,8 @@ public class ReanalysisRequiredStatusService {
                                                            boolean ignoreProcessingStates,
                                                            Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
                                                            Map<String, Long> dossierVersionMap,
-                                                           Map<String, DossierEntity> dossierMap) {
+                                                           Map<String, DossierEntity> dossierMap,
+                                                           Map<String, Map<String, Integer>> componentMappingVersionMap) {
 
         // enhance with dossierTemplateId
         DossierEntity dossier;
@@ -101,7 +104,7 @@ public class ReanalysisRequiredStatusService {
         }
 
         if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus()) && !ignoreProcessingStates) {
-            return new AnalysisRequiredResult(false, false);
+            return computeAnalysisRequiredForErrorState(dossier, fileStatus, dossierTemplateVersionMap);
         }
 
         if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus())
@@ -118,7 +121,7 @@ public class ReanalysisRequiredStatusService {
                     if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) {
                         return new AnalysisRequiredResult(true, false);
                     } else {
-                        return requiresReanalysisBasedOnVersionDifference(fileStatus, dossier, dossierTemplateVersionMap, dossierVersionMap);
+                        return requiresReanalysisBasedOnVersionDifference(fileStatus, dossier, dossierTemplateVersionMap, dossierVersionMap,componentMappingVersionMap);
                     }
                 default:
                     return new AnalysisRequiredResult(false, false);
@@ -129,18 +132,35 @@ public class ReanalysisRequiredStatusService {
     }
 
 
+    private AnalysisRequiredResult computeAnalysisRequiredForErrorState(DossierEntity dossier,FileModel fileStatus, Map<String, Map<VersionType, Long>> dossierTemplateVersionMap){
+        Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
+                                                                                                   k -> buildVersionData(dossier.getDossierTemplateId()));
+        var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L);
+        var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L);
+
+        var isRuleExecutionTimeout = fileStatus.getFileErrorInfo() != null && fileStatus.getFileErrorInfo().getErrorCode() != null && fileStatus.getFileErrorInfo().getErrorCode().equals(
+                ErrorCode.RULES_EXECUTION_TIMEOUT);
+
+        var fullAnalysisRequired = (!rulesVersionMatches || !componentRulesVersionMatches) && !isRuleExecutionTimeout;
+
+        return new AnalysisRequiredResult(false, fullAnalysisRequired);
+    }
+
+
     private AnalysisRequiredResult requiresReanalysisBasedOnVersionDifference(FileModel fileStatus,
                                                                               DossierEntity dossier,
                                                                               Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
-                                                                              Map<String, Long> dossierVersionMap) {
+                                                                              Map<String, Long> dossierVersionMap,
+                                                                              Map<String, Map<String,Integer>> componentMappingVersionMap) {
 
         // get relevant versions
         Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
                                                                                                    k -> buildVersionData(dossier.getDossierTemplateId()));
 
-        Map<String, Integer> componentMappingVersions = componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId())
-                .stream()
-                .collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion));
+        Map<String, Integer> componentMappingVersions = componentMappingVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
+                                                                                                   k -> componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId())
+                                                                                                           .stream()
+                                                                                                           .collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion)));
 
         Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId()));
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java

@@ -173,11 +173,11 @@ public class ReanalysisService {
         relevantFiles.stream()
                 .filter(fileStatus -> fileStatus.getOcrStartTime() == null)
                 .filter(fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.PROCESSED))
-                .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId()));
+                .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId(), false));
     }
 
 
-    public void ocrFile(String dossierId, String fileId, boolean force) {
+    public void ocrFile(String dossierId, String fileId, boolean force, boolean allPages) {
 
         dossierPersistenceService.getAndValidateDossier(dossierId);
         FileModel dossierFile = fileStatusService.getStatus(fileId);
@@ -197,18 +197,18 @@ public class ReanalysisService {
         }
 
         if (force) {
-            fileStatusService.setStatusOcrQueued(dossierId, fileId);
+            fileStatusService.setStatusOcrQueued(dossierId, fileId, allPages);
         } else {
             if (dossierFile.getOcrStartTime() != null) {
                 throw new ConflictException("File already has been OCR processed");
             }
 
-            ocrFiles(dossierId, Sets.newHashSet(fileId));
+            ocrFiles(dossierId, Sets.newHashSet(fileId), allPages);
         }
     }
 
 
-    public void ocrFiles(String dossierId, Set<String> fileIds) {
+    public void ocrFiles(String dossierId, Set<String> fileIds, boolean allPages) {
 
         var relevantFiles = getRelevantFiles(dossierId, fileIds);
 
@@ -220,7 +220,7 @@ public class ReanalysisService {
 
         relevantFiles.stream()
                 .filter(fileStatus -> fileStatus.getOcrStartTime() == null)
-                .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId()));
+                .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId(), allPages));
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java

@@ -14,7 +14,6 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
 import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
@@ -38,7 +37,6 @@ public class AutomaticAnalysisJob implements Job {
     private final FileStatusService fileStatusService;
     private final TenantProvider tenantProvider;
     private final ObservationRegistry observationRegistry;
-    private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
 
     @Setter
     private boolean schedulingStopped;
@@ -69,11 +67,6 @@ public class AutomaticAnalysisJob implements Job {
 
                     TenantContext.setTenantId(tenant.getTenantId());
 
-                    if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) {
-                        log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId());
-                        return;
-                    }
-
                     String queueName = MessagingConfiguration.REDACTION_REQUEST_QUEUE_PREFIX + "_" + tenant.getTenantId();
                     var redactionQueueInfo = amqpAdmin.getQueueInfo(queueName);
                     if (redactionQueueInfo != null) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java

@@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionEntrySource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
@@ -108,7 +109,7 @@ public class ManualRedactionMapper {
 
             for (EntityLogEntry entityLogEntry : entityLogEntriesWithId) {
 
-                if (invalidDictionaryRequest(removeRedactionRequest, entityLogEntry) || invalidLocalRequest(removeRedactionRequest, entityLogEntry)) {
+                if (entityLogEntry.getState().equals(EntryState.PENDING) || entityLogEntry.getState().equals(EntryState.REMOVED)) {
                     continue;
                 }
 
@@ -135,28 +136,10 @@ public class ManualRedactionMapper {
     }
 
 
-    private static boolean invalidLocalRequest(RemoveRedactionRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) {
-
-        return !isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.REMOVED);
-    }
-
-
-    private static boolean isDictionaryRequest(RemoveRedactionRequestModel removeRedactionRequest) {
-
-        return removeRedactionRequest.isRemoveFromDictionary() || removeRedactionRequest.isRemoveFromAllDossiers();
-    }
-
-
-    private static boolean invalidDictionaryRequest(RemoveRedactionRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) {
-
-        return isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.PENDING);
-    }
-
-
     public List<RequestEntryPair<ForceRedactionRequest>> toForceRedactionRequestList(String dossierId,
                                                                                      String fileId,
                                                                                      Set<ForceRedactionRequestModel> forceRedactionRequests,
-                                                                                     Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
+                                                                                     Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
 
         List<RequestEntryPair<ForceRedactionRequest>> requests = new ArrayList<>();
 
@@ -172,9 +155,8 @@ public class ManualRedactionMapper {
                     .build();
 
             if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !entityLogEntry.getEngines().contains(Engine.IMPORTED) && entryIsEntityType(entityLogEntry)) {
-                request.setBasedOnDictAnnotationId(forceRedactionRequestModel.getAnnotationId());
                 entityLogEntry.setId(uuid);
-                manualRedactionEntryConsumer.accept(entityLogEntry);
+                manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, forceRedactionRequestModel.getAnnotationId()));
                 request.setAnnotationId(uuid);
             }
 
@@ -188,7 +170,7 @@ public class ManualRedactionMapper {
     public List<RequestEntryPair<LegalBasisChangeRequest>> toLegalBasisChangeRequestList(String dossierId,
                                                                                          String fileId,
                                                                                          Set<LegalBasisChangeRequestModel> legalBasisChangeRequests,
-                                                                                         Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
+                                                                                         Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
 
         List<RequestEntryPair<LegalBasisChangeRequest>> requests = new ArrayList<>();
 
@@ -206,9 +188,8 @@ public class ManualRedactionMapper {
                     .build();
 
             if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !entityLogEntry.getEngines().contains(Engine.IMPORTED) && entryIsEntityType(entityLogEntry)) {
-                request.setBasedOnDictAnnotationId(legalBasisChangeRequest.getAnnotationId());
                 entityLogEntry.setId(uuid);
-                manualRedactionEntryConsumer.accept(entityLogEntry);
+                manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, legalBasisChangeRequest.getAnnotationId()));
                 request.setAnnotationId(uuid);
             }
 
@@ -224,7 +205,7 @@ public class ManualRedactionMapper {
                                                                                          String dossierTemplateId,
                                                                                          Set<RecategorizationRequestModel> recategorizationRequests,
                                                                                          boolean includeUnprocessed,
-                                                                                         Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
+                                                                                         Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
 
         List<EntityLogEntry> entityLogEntries = entityLogMongoWrapperService.getEntityLogEntriesByIds(dossierId,
                                                                                                       fileId,
@@ -247,7 +228,7 @@ public class ManualRedactionMapper {
 
             for (EntityLogEntry entityLogEntry : entityLogEntriesById) {
 
-                if (invalidDictionaryRequest(recategorizationRequest, entityLogEntry) || invalidLocalRequest(recategorizationRequest, entityLogEntry)) {
+                if (entityLogEntry.getState().equals(EntryState.PENDING) || entityLogEntry.getState().equals(EntryState.REMOVED)) {
                     continue;
                 }
 
@@ -291,9 +272,8 @@ public class ManualRedactionMapper {
                     && !recategorizationRequest.isAddToAllDossiers()
                     && !recategorizationRequest.isAddToDictionary()
                     && entryIsEntityType(entityLogEntry)) {
-                    request.setBasedOnDictAnnotationId(recategorizationRequest.getAnnotationId());
                     entityLogEntry.setId(uuid);
-                    manualRedactionEntryConsumer.accept(entityLogEntry);
+                    manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, recategorizationRequest.getAnnotationId()));
                     request.setAnnotationId(uuid);
                 }
 
@@ -305,24 +285,6 @@ public class ManualRedactionMapper {
     }
 
 
-    private static boolean invalidLocalRequest(RecategorizationRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) {
-
-        return !isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.REMOVED);
-    }
-
-
-    private static boolean isDictionaryRequest(RecategorizationRequestModel removeRedactionRequest) {
-
-        return removeRedactionRequest.isAddToDictionary() || removeRedactionRequest.isAddToAllDossiers();
-    }
-
-
-    private static boolean invalidDictionaryRequest(RecategorizationRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) {
-
-        return isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.PENDING);
-    }
-
-
     private void checkSectionLength(String changedSection) {
 
         if (changedSection == null) {
@@ -353,7 +315,7 @@ public class ManualRedactionMapper {
     public List<RequestEntryPair<ResizeRedactionRequest>> toResizeRedactionRequestList(String dossierId,
                                                                                        String fileId,
                                                                                        Set<ResizeRedactionRequestModel> resizeRedactionRequests,
-                                                                                       Consumer<EntityLogEntry> manualRedactionEntryConsumer,
+                                                                                       Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer,
                                                                                        boolean includeUnprocessed) {
 
         List<EntityLogEntry> entityLogEntries = entityLogMongoWrapperService.getEntityLogEntriesByIds(dossierId,
@@ -366,8 +328,11 @@ public class ManualRedactionMapper {
         List<RequestEntryPair<ResizeRedactionRequest>> requests = new ArrayList<>();
 
         for (ResizeRedactionRequestModel resizeRedactionRequest : resizeRedactionRequests) {
+            for (EntityLogEntry entityLogEntry : entityLogEntries) {
 
-            entityLogEntries.forEach(entityLogEntry -> {
+                if (entityLogEntry.getState().equals(EntryState.PENDING) || entityLogEntry.getState().equals(EntryState.REMOVED)) {
+                    continue;
+                }
                 String uuid = UUID.randomUUID().toString();
 
                 ResizeRedactionRequest request = ResizeRedactionRequest.builder()
@@ -385,14 +350,13 @@ public class ManualRedactionMapper {
                     && !request.isAddToAllDossiers()
                     && !request.getUpdateDictionary()
                     && entryIsEntityType(entityLogEntry)) {
-                    request.setBasedOnDictAnnotationId(resizeRedactionRequest.getAnnotationId());
                     entityLogEntry.setId(uuid);
-                    manualRedactionEntryConsumer.accept(entityLogEntry);
+                    manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, resizeRedactionRequest.getAnnotationId()));
                     request.setAnnotationId(uuid);
                 }
 
                 requests.add(RequestEntryPair.<ResizeRedactionRequest>builder().request(request).entityLogEntry(entityLogEntry).build());
-            });
+            }
 
         }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java

@@ -58,6 +58,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionEntrySource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
@@ -637,8 +638,9 @@ public class ManualRedactionService {
     }
 
 
-    private void addManualRedactionEntry(String fileId, EntityLogEntry entityLogEntry) {
+    private void addManualRedactionEntry(String fileId, ManualRedactionEntrySource manualRedactionEntrySource) {
 
+        EntityLogEntry entityLogEntry = manualRedactionEntrySource.getEntityLogEntry();
         ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
                 .value(entityLogEntry.getValue())
                 .reason(entityLogEntry.getReason())
@@ -656,6 +658,7 @@ public class ManualRedactionService {
                 .dictionaryEntryType(getDictionaryEntryType(entityLogEntry))
                 .fileId(fileId)
                 .requestDate(OffsetDateTime.now())
+                .sourceId(manualRedactionEntrySource.getSourceId())
                 .build();
 
         addManualRedactionEntries(List.of(manualRedactionEntry), false);
@@ -670,7 +673,7 @@ public class ManualRedactionService {
     }
 
 
-    private Consumer<EntityLogEntry> getEntityLogEntryConsumer(String fileId) {
+    private Consumer<ManualRedactionEntrySource> getEntityLogEntryConsumer(String fileId) {
 
         return entry -> addManualRedactionEntry(fileId, entry);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/PendingEntryFactory.java

@@ -94,7 +94,7 @@ public class PendingEntryFactory {
                 .endOffset(-1)
                 .changes(Collections.emptyList())
                 .manualChanges(manualChanges)
-                .engines(new HashSet<>(Set.of(Engine.DICTIONARY)))
+                .engines(new HashSet<>(Set.of(manualRedactionEntry.isAddToDossierDictionary() ? Engine.DOSSIER_DICTIONARY : Engine.DICTIONARY)))
                 .reference(Collections.emptySet())
                 .importedRedactionIntersections(Collections.emptySet())
                 .build();

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierAttributePersistenceService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
 
+import java.util.Collection;
 import java.util.List;
 
 import jakarta.transaction.Transactional;
@@ -58,6 +59,11 @@ public class DossierAttributePersistenceService {
         return dossierAttributeRepository.findByIdDossierId(dossierId);
     }
 
+    public List<DossierAttributeEntity> getDossierAttributes(Collection<String> dossierIds) {
+
+        return dossierAttributeRepository.findByDossierIds(dossierIds);
+    }
+
 
     public DossierAttributeEntity findOne(String dossierId, String dossierAttributeId) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java

@@ -4,6 +4,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.exceptio
 
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -23,6 +24,9 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntryV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileChangeEntryV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierChange;
 
@@ -179,7 +183,7 @@ public class DossierPersistenceService {
     }
 
 
-    public List<DossierEntity> findAllDossiers(List<String> dossierIds) {
+    public List<DossierEntity> findAllDossiers(Collection<String> dossierIds) {
 
         if (!dossierIds.isEmpty()) {
             return dossierRepository.findAllById(dossierIds);
@@ -195,9 +199,9 @@ public class DossierPersistenceService {
     }
 
 
-    public Set<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
 
-        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId);
+        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId, dossierIds);
     }
 
 
@@ -232,6 +236,19 @@ public class DossierPersistenceService {
     }
 
 
+    public DossierChangeResponseV2 hasChangesSinceV2(OffsetDateTime since) {
+
+        var changedDossiers = dossierRepository.findDossierChangeProjectionByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS));
+        var changedFiles = fileRepository.findFileChangeProjectionByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS));
+
+        var response = new DossierChangeResponseV2();
+        response.setDossierChanges(changedDossiers.stream().map(d -> new DossierChangeEntryV2(d.getId(), d.getLastUpdated())).collect(Collectors.toList()));
+        response.setFileChanges(changedFiles.stream().map(f -> new FileChangeEntryV2(f.getId(), f.getDossierId(), f.getLastUpdated())).collect(Collectors.toList()));
+
+        return response;
+    }
+
+
     public Set<DossierChange> hasChangesSince(OffsetDateTime since) {
 
         var dossiersWithChanges = dossierRepository.findDossierChangeByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS));
@@ -282,4 +299,10 @@ public class DossierPersistenceService {
                 .orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
     }
 
+
+    public List<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
+
+        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileAttributeConfigPersistenceService.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
 
-import static com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService.ASCII_ENCODING;
-import static com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService.ISO_ENCODING;
-import static com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService.UTF_ENCODING;
+import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ASCII_ENCODING;
+import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ISO_ENCODING;
+import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.UTF_ENCODING;
 
 import java.util.List;
 import java.util.Objects;
@@ -81,7 +81,7 @@ public class FileAttributeConfigPersistenceService {
 
 
     @Transactional
-    public List<FileAttributeConfigEntity> setFileAttributesConfig(String dossierTemplateId, List<FileAttributeConfigEntity> fileAttributesConfig) {
+    public void setFileAttributesConfig(String dossierTemplateId, List<FileAttributeConfigEntity> fileAttributesConfig) {
 
         Set<String> toSetIds = fileAttributesConfig.stream()
                 .map(FileAttributeConfigEntity::getId)
@@ -98,8 +98,6 @@ public class FileAttributeConfigPersistenceService {
             fileAttributesRepository.deleteByFileAttributeConfigId(ctr.getId());
             fileAttributeConfigRepository.deleteById(ctr.getId());
         });
-
-        return getFileAttributes(dossierTemplateId);
     }