RED-10422 - improved number of requests with help of new changes endpoint

RED-4732: Fixed some auditlog messages

See merge request redactmanager/persistence-service!844

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ApplicationConfigurationController.java

@@ -9,11 +9,17 @@ import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.ApplicationConfigurationResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
+import jakarta.persistence.Column;
 import jakarta.validation.Valid;
+import lombok.Builder;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -23,14 +29,24 @@ import lombok.extern.slf4j.Slf4j;
 public class ApplicationConfigurationController implements ApplicationConfigurationResource {
 
     private final ApplicationConfigService applicationConfigService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
     @PreAuthorize("hasAuthority('" + WRITE_APP_CONFIG + "')")
     public ApplicationConfig createOrUpdateAppConfig(@Valid @RequestBody ApplicationConfig appConfig) {
 
-        return MagicConverter.convert(applicationConfigService.saveApplicationConfiguration(MagicConverter.convert(appConfig, ApplicationConfigurationEntity.class)),
+        var result = MagicConverter.convert(applicationConfigService.saveApplicationConfiguration(convert(appConfig)),
                                       ApplicationConfig.class);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId("ApplicationConfig")
+                                              .category(AuditCategory.SETTINGS.name())
+                                              .message("Application config has been changed.")
+                                              .build());
+
+        return result;
     }
 
 
@@ -41,4 +57,18 @@ public class ApplicationConfigurationController implements ApplicationConfigurat
         return MagicConverter.convert(applicationConfigService.getApplicationConfig(), ApplicationConfig.class);
     }
 
+
+    private ApplicationConfigurationEntity convert(ApplicationConfig appConfig){
+        var entity = ApplicationConfigurationEntity.builder()
+                .downloadCleanupDownloadFilesHours(appConfig.getDownloadCleanupDownloadFilesHours())
+                .downloadCleanupNotDownloadFilesHours(appConfig.getDownloadCleanupNotDownloadFilesHours())
+                .softDeleteCleanupTime(appConfig.getSoftDeleteCleanupTime())
+                .build();
+
+        if(appConfig.getHardDeleteCleanupRetryTime() != null){
+            entity.setHardDeleteCleanupRetryTime(appConfig.getHardDeleteCleanupRetryTime());
+        }
+        return entity;
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/CustomPermissionMappingController.java

@@ -11,8 +11,12 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.acl.custom.service.CustomPermissionService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.CustomPermissionMappingResource;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.permission.CustomPermissionMappingModel;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import lombok.RequiredArgsConstructor;
 
@@ -21,6 +25,7 @@ import lombok.RequiredArgsConstructor;
 public class CustomPermissionMappingController implements CustomPermissionMappingResource {
 
     private final CustomPermissionService customPermissionService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
@@ -36,6 +41,14 @@ public class CustomPermissionMappingController implements CustomPermissionMappin
     public void saveCustomPermissionMappings(@PathVariable(TARGET_OBJECT_NAME) String targetObject, @RequestBody List<CustomPermissionMappingModel> customPermissionMappingModels) {
 
         customPermissionService.saveCustomPermissionMappings(targetObject, customPermissionMappingModels);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(targetObject)
+                                              .category(AuditCategory.SETTINGS.name())
+                                              .message("Custom permissions have been changed.")
+                                              .build());
+
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java

@@ -11,6 +11,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -23,7 +24,11 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.model.websocket.DossierEventType;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
 
+import com.iqser.red.service.persistence.management.v1.processor.service.FilterByPermissionsService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.JsonNode;
 import org.apache.commons.lang3.StringUtils;
+
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -76,6 +81,7 @@ public class DossierController implements DossierResource {
 
     private final DossierManagementService dossierManagementService;
     private final UserService userService;
+    private final FilterByPermissionsService filterByPermissionsService;
     private final FileStatusManagementService fileStatusManagementService;
     private final AuditPersistenceService auditPersistenceService;
     private final NotificationPersistenceService notificationPersistenceService;
@@ -106,6 +112,20 @@ public class DossierController implements DossierResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
+    public DossierChangeResponseV2 changesSinceV2(@RequestBody JSONPrimitive<OffsetDateTime> since) {
+
+        DossierChangeResponseV2 changes = dossierManagementService.changesSinceV2(since);
+
+        // filter only viewables
+        changes.setFileChanges(filterByPermissionsService.onlyViewableHavingDossierId(changes.getFileChanges()));
+        changes.setDossierChanges(filterByPermissionsService.onlyViewableHavingDossierId(changes.getDossierChanges()));
+
+        return changes;
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DOSSIER + "') && (#dossierRequest.dossierId == null || hasPermission(#dossierRequest.dossierId, 'Dossier', 'ACCESS_OBJECT') )")
     public ResponseEntity<Dossier> createDossierOrUpdateDossier(@RequestBody DossierRequest dossierRequest) {
@@ -405,6 +425,28 @@ public class DossierController implements DossierResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
+    public JSONPrimitive<Map<String, Dossier>> getDossiersByIds(@RequestBody JSONPrimitive<Set<String>> dossierIds) {
+
+        // filter dossiers based on view
+        var viewableDossierIds = filterByPermissionsService.onlyViewableDossierIds(dossierIds.getValue());
+        // load dossiers
+        var dossiers = dossierManagementService.getDossiersByIds(viewableDossierIds);
+
+        // add attributes and ACL - already filtered before loading
+        enhanceDossiersWithAttributeAndACLData(dossiers,false);
+
+        // build response
+        var responseMap = new LinkedHashMap<String, Dossier>();
+        for (var dossier : dossiers) {
+            responseMap.put(dossier.getId(), dossier);
+        }
+
+        return new JSONPrimitive<>(responseMap);
+    }
+
+
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
     public Dossier getDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
                               @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
@@ -418,70 +460,45 @@ public class DossierController implements DossierResource {
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                                      @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted) {
 
-        var dossiers = dossierManagementService.getAllDossiers(includeArchived, includeDeleted)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getAllDossiers(includeArchived, includeDeleted);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getDossiersForDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
                                                        @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                                                        @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted) {
 
-        var dossiers = dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getSoftDeletedDossiers() {
 
-        var dossiers =  dossierManagementService.getSoftDeletedDossiers()
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getSoftDeletedDossiers();
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiers() {
 
-        var dossiers =  dossierManagementService.getArchivedDossiers()
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getArchivedDossiers();
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiersForDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) {
 
-        var dossiers = dossierManagementService.getArchivedDossiersForDossierTemplateId(dossierTemplateId)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getArchivedDossiersForDossierTemplateId(dossierTemplateId);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
@@ -586,5 +603,31 @@ public class DossierController implements DossierResource {
         return new DossierAttributes(attributeIdToValue);
     }
 
+    private List<Dossier> enhanceDossiersWithAttributeAndACLData(List<Dossier> dossiers) {
+
+            return enhanceDossiersWithAttributeAndACLData(dossiers, true);
+    }
+
+    private List<Dossier> enhanceDossiersWithAttributeAndACLData(List<Dossier> dossiers, boolean filter) {
+
+        // filter first, only load attributes and ACL for viewable dossiers
+        List<Dossier> filteredDossiers = filter ? filterByPermissionsService.onlyViewableDossiers(dossiers) : dossiers;
+
+        // load all attributes at once
+        var attributes = dossierAttributePersistenceService.getDossierAttributes(filteredDossiers.stream().map(Dossier::getId).collect(Collectors.toSet()));
+        var attributesMap = new HashMap<String, List<DossierAttributeEntity>>();
+        for (DossierAttributeEntity attribute : attributes) {
+            attributesMap.computeIfAbsent(attribute.getId().getDossierId(), k -> new ArrayList<>()).add(attribute);
+        }
+
+        for (var dossier : filteredDossiers) {
+            // set attributes
+            dossier.setDossierAttributes(convertDossierAttributes(attributesMap.getOrDefault(dossier.getId(), new ArrayList<>())));
+            // set ACL data
+            dossierACLService.enhanceDossierWithACLData(dossier);
+        }
+        return filteredDossiers;
+    }
+
 }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierStatusController.java

@@ -14,13 +14,17 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.ColorUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.DossierStatusResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -31,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class DossierStatusController implements DossierStatusResource {
 
     private final DossierStatusPersistenceService dossierStatusPersistenceService;
+    private final AuditPersistenceService auditPersistenceService;
 
 
     @Override
@@ -57,6 +62,15 @@ public class DossierStatusController implements DossierStatusResource {
                                                                                            .rank(dossierStatusRequest.getRank())
                                                                                            .color(dossierStatusRequest.getColor())
                                                                                            .build());
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(dossierStatusRequest.getDossierTemplateId())
+                                              .category(AuditCategory.DOSSIER_TEMPLATE.name())
+                                              .message("Dossier states have been updated.")
+                                              .build());
+
+
         return MagicConverter.convert(response, DossierStatusInfo.class);
     }
 
@@ -91,7 +105,17 @@ public class DossierStatusController implements DossierStatusResource {
     public void deleteDossierStatus(@PathVariable("dossierStatusId") String dossierStatusId,
                                     @RequestParam(value = DOSSIER_STATUS_REPLACE_ID, required = false) String replaceDossierStatusId) {
 
+        var dossierTemplateId = dossierStatusPersistenceService.getDossierStatus(dossierStatusId).getDossierTemplateId();
+
         dossierStatusPersistenceService.deleteDossierStatus(dossierStatusId, replaceDossierStatusId);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(dossierTemplateId)
+                                              .category(AuditCategory.DOSSIER_TEMPLATE.name())
+                                              .message("Dossier state has been deleted.")
+                                              .build());
+
     }
 
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java

@@ -252,8 +252,7 @@ public class ManualRedactionController implements ManualRedactionResource {
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse removeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                        @PathVariable(FILE_ID) String fileId,
-                                                       @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests,
-                                                       @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                       @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests) {
 
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
         verifyAccessForDossier(dossierId,
@@ -265,7 +264,7 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                                                                 fileId,
                                                                                                 removeRedactionRequests,
                                                                                                 dossier.getDossierTemplateId(),
-                                                                                                includeUnprocessed);
+                                                                                                true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -282,8 +281,7 @@ public class ManualRedactionController implements ManualRedactionResource {
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse removeRedactionBulkLocal(String dossierId,
                                                             String fileId,
-                                                            RemoveRedactionBulkLocalRequestModel removeRedactionRequest,
-                                                            boolean includeUnprocessed) {
+                                                            RemoveRedactionBulkLocalRequestModel removeRedactionRequest) {
 
         verifyAccess(dossierId, fileId);
         verifyRequest(removeRedactionRequest.isRectangle(), removeRedactionRequest.getPosition(), removeRedactionRequest.getValue());
@@ -309,7 +307,7 @@ public class ManualRedactionController implements ManualRedactionResource {
 
             List<EntityLogEntryResponse> filteredEntityLogResponses = getFilteredEntityLogResponses(dossierId,
                                                                                                     fileId,
-                                                                                                    includeUnprocessed,
+                                                                                                    true,
                                                                                                     removeRedactionRequest.isRectangle(),
                                                                                                     removeRedactionRequest.getValue(),
                                                                                                     removeRedactionRequest.isCaseSensitive(),
@@ -322,7 +320,7 @@ public class ManualRedactionController implements ManualRedactionResource {
                     .map(entityLogEntry -> RemoveRedactionRequestModel.builder().annotationId(entityLogEntry.getId()).comment(removeRedactionRequest.getComment()).build())
                     .collect(Collectors.toSet());
         }
-        return removeRedactionBulk(dossierId, fileId, removeRedactionRequestModels, includeUnprocessed);
+        return removeRedactionBulk(dossierId, fileId, removeRedactionRequestModels);
     }
 
 
@@ -371,13 +369,12 @@ public class ManualRedactionController implements ManualRedactionResource {
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse recategorizeBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                     @PathVariable(FILE_ID) String fileId,
-                                                    @RequestBody Set<RecategorizationRequestModel> recategorizationRequests,
-                                                    @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                    @RequestBody Set<RecategorizationRequestModel> recategorizationRequests) {
 
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
         verifyAccess(dossierId, fileId);
 
-        List<ManualAnnotationResponse> responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, includeUnprocessed);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -394,8 +391,7 @@ public class ManualRedactionController implements ManualRedactionResource {
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse recategorizeBulkLocal(String dossierId,
                                                          String fileId,
-                                                         RecategorizationBulkLocalRequestModel recategorizationRequest,
-                                                         boolean includeUnprocessed) {
+                                                         RecategorizationBulkLocalRequestModel recategorizationRequest) {
 
         verifyAccess(dossierId, fileId);
         verifyRequest(recategorizationRequest.isRectangle(), recategorizationRequest.getPosition(), recategorizationRequest.getValue());
@@ -428,7 +424,7 @@ public class ManualRedactionController implements ManualRedactionResource {
 
             List<EntityLogEntryResponse> filteredEntityLogResponses = getFilteredEntityLogResponses(dossierId,
                                                                                                     fileId,
-                                                                                                    includeUnprocessed,
+                                                                                                    true,
                                                                                                     recategorizationRequest.isRectangle(),
                                                                                                     recategorizationRequest.getValue(),
                                                                                                     recategorizationRequest.isCaseSensitive(),
@@ -449,19 +445,18 @@ public class ManualRedactionController implements ManualRedactionResource {
                     .collect(Collectors.toSet());
         }
 
-        return recategorizeBulk(dossierId, fileId, recategorizationRequestModels, includeUnprocessed);
+        return recategorizeBulk(dossierId, fileId, recategorizationRequestModels);
     }
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public ManualRedactionResponse resizeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                        @PathVariable(FILE_ID) String fileId,
-                                                       @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests,
-                                                       @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
+                                                       @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests) {
 
         verifyAccessAndDossierExistence(dossierId, fileId);
 
-        List<ManualAnnotationResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, includeUnprocessed);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RulesController.java

@@ -26,6 +26,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.FileU
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.RulesValidationMapper;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.RulesResource;
@@ -52,6 +53,7 @@ public class RulesController implements RulesResource {
     private final RulesPersistenceService rulesPersistenceService;
     private final RulesValidationService rulesValidationService;
     private final AuditPersistenceService auditPersistenceService;
+    private final FileStatusPersistenceService fileStatusPersistenceService;
 
 
     @Override
@@ -75,6 +77,7 @@ public class RulesController implements RulesResource {
         }
         if (!rules.isDryRun()) {
             rulesPersistenceService.setRules(rulesUploadRequest.getRules(), rulesUploadRequest.getDossierTemplateId(), rulesUploadRequest.getRuleFileType());
+            fileStatusPersistenceService.resetErrorCounter(rules.getDossierTemplateId());
         }
 
         auditPersistenceService.audit(AuditRequest.builder()

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java

@@ -14,11 +14,16 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
 import org.springframework.security.access.AccessDeniedException;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService;
@@ -30,6 +35,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Approva
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
+import com.iqser.red.service.persistence.management.v1.processor.service.FilterByPermissionsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
@@ -39,6 +45,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
@@ -46,6 +53,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.notificatio
 import com.iqser.red.service.persistence.service.v1.api.shared.model.warning.ApproveResponse;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -67,6 +77,7 @@ public class StatusController implements StatusResource {
     private final NotificationPersistenceService notificationPersistenceService;
     private final DossierACLService dossierACLService;
     private final ApprovalVerificationService approvalVerificationService;
+    private final FilterByPermissionsService filterByPermissionsService;
 
 
     @Override
@@ -82,6 +93,23 @@ public class StatusController implements StatusResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
+    public JSONPrimitive<Map<String, List<FileStatus>>> getFilesByIds(@RequestBody JSONPrimitive<Map<String, Set<String>>> filesByDossier) {
+
+        // filter dossiers by view
+        var accessibleDossierIds = filterByPermissionsService.onlyViewableDossierIds(new ArrayList<>(filesByDossier.getValue().keySet()));
+        var response = new HashMap<String, List<FileStatus>>();
+        for (var dossierId : accessibleDossierIds) {
+            var allFoundFiles = fileStatusManagementService.findAllDossierIdAndIds(dossierId, filesByDossier.getValue().get(dossierId));
+            response.put(dossierId, allFoundFiles.stream().map(FileStatusMapper::toFileStatus).collect(Collectors.toList()));
+        }
+
+        return new JSONPrimitive<>(response);
+
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
     public Map<String, List<FileStatus>> getDossierStatus(@RequestBody List<String> dossierIds) {

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -21,6 +21,7 @@ import org.springframework.web.multipart.MultipartFile;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService;
+import com.iqser.red.service.persistence.management.v1.processor.migration.MigrationController;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
@@ -156,6 +157,7 @@ public class SupportController implements SupportResource {
         return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
     }
 
+
     @Override
     @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
     public ImportResponse importDataset(MultipartFile file) {

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java

@@ -269,7 +269,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     @Override
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
-    public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter) {
+    public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter, String quoteChar) {
 
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
@@ -285,18 +285,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
             throw new BadRequestException(format("The provided file name \"%s\" is not valid!", nameToUse));
         }
 
-        if (Strings.isNullOrEmpty(delimiter)) {
-            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
-        } else if (delimiter.length() != 1) {
-            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
-        }
-        char cleanDelimiter = delimiter.charAt(0);
+        char cleanDelimiter = getDelimiter(delimiter);
+        char cleanQuoteChar = getQuoteChar(quoteChar);
 
         Path mappingFile = saveToFile(file);
 
         try {
 
-            ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, cleanDelimiter, encoding, mappingFile.toFile());
+            ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
+                                                                               nameToUse,
+                                                                               fileName,
+                                                                               cleanDelimiter,
+                                                                               encoding,
+                                                                               mappingFile.toFile(),
+                                                                               cleanQuoteChar);
 
             return componentMappingMapper.toModel(metaData);
         } finally {
@@ -309,18 +311,20 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     @Override
     @SneakyThrows
     @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
-    public ComponentMappingMetadataModel updateMapping(String dossierTemplateId, String componentMappingId, MultipartFile file, String name, String encoding, String delimiter) {
+    public ComponentMappingMetadataModel updateMapping(String dossierTemplateId,
+                                                       String componentMappingId,
+                                                       MultipartFile file,
+                                                       String name,
+                                                       String encoding,
+                                                       String delimiter,
+                                                       String quoteChar) {
 
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
         String nameToUse = validateFileName(file, name);
 
-        if (Strings.isNullOrEmpty(delimiter)) {
-            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
-        } else if (delimiter.length() != 1) {
-            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
-        }
-        char cleanDelimiter = delimiter.charAt(0);
+        char cleanDelimiter = getDelimiter(delimiter);
+        char cleanQuoteChar = getQuoteChar(quoteChar);
 
         Path mappingFile = saveToFile(file);
 
@@ -331,7 +335,8 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
                                                                                      encoding,
                                                                                      cleanDelimiter,
                                                                                      mappingFile.toFile(),
-                                                                                     file.getOriginalFilename());
+                                                                                     file.getOriginalFilename(),
+                                                                                     cleanQuoteChar);
 
             return componentMappingMapper.toModel(resultMetaData);
         } finally {
@@ -340,6 +345,28 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
     }
 
 
+    private static char getDelimiter(String delimiter) {
+
+        if (Strings.isNullOrEmpty(delimiter)) {
+            throw new BadRequestException("The provided delimiter is not valid! Can't be null or empty.");
+        } else if (delimiter.length() != 1) {
+            throw new BadRequestException(format("The provided delimiter %s is not valid! Only a single character is allowed.", delimiter));
+        }
+        return delimiter.charAt(0);
+    }
+
+
+    private static char getQuoteChar(String quoteChar) {
+
+        if (Strings.isNullOrEmpty(quoteChar)) {
+            throw new BadRequestException("The provided quoteChar is not valid! Can't be null or empty.");
+        } else if (quoteChar.length() != 1) {
+            throw new BadRequestException(format("The provided quoteChar %s is not valid! Only a single character is allowed.", quoteChar));
+        }
+        return quoteChar.charAt(0);
+    }
+
+
     private static String validateFileName(MultipartFile file, String name) {
 
         if (Strings.isNullOrEmpty(file.getOriginalFilename()) || !file.getOriginalFilename().endsWith(".csv")) {

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DossierResource.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.external.resource;
 
 import java.time.OffsetDateTime;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.springframework.http.HttpStatus;
@@ -17,6 +18,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierInformation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
@@ -29,10 +31,12 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
 public interface DossierResource {
 
     String DOSSIER_REST_PATH = ExternalApi.BASE_PATH + "/dossier";
+    String BY_ID_PATH = "/by-id";
     String DOSSIER_TEMPLATE_PATH = "/dossier-template";
     String DOSSIER_INFO_PATH = "/info";
     String DELETED_DOSSIERS_PATH = ExternalApi.BASE_PATH + "/deleted-dossiers";
     String CHANGES_DETAILS_PATH = "/changes/details";
+    String CHANGES_DETAILS_V2_PATH = "/changes/details/v2";
     String HARD_DELETE_PATH = "/hard-delete";
     String UNDELETE_PATH = "/restore";
 
@@ -62,6 +66,12 @@ public interface DossierResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success")})
     List<DossierChangeEntry> changesSince(@RequestBody JSONPrimitive<OffsetDateTime> since);
 
+    @ResponseBody
+    @PostMapping(value = DOSSIER_REST_PATH + CHANGES_DETAILS_V2_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "See if there are changes to dossiers since param", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success")})
+    DossierChangeResponseV2 changesSinceV2(@RequestBody JSONPrimitive<OffsetDateTime> since);
+
 
     @ResponseBody
     @PostMapping(value = DOSSIER_REST_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@@ -95,6 +105,13 @@ public interface DossierResource {
     List<Dossier> getDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                               @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted);
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = DOSSIER_REST_PATH+BY_ID_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets dossiers by ids.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    JSONPrimitive<Map<String,Dossier>> getDossiersByIds(@RequestBody JSONPrimitive<Set<String>> dossierIds);
+
 
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DossierTemplateResource.java

@@ -95,35 +95,36 @@ public interface DossierTemplateResource {
 
     @ResponseStatus(value = HttpStatus.NO_CONTENT)
     @PostMapping(value = DOSSIER_TEMPLATE_PATH + IMPORT_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Receives an archive to import", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Archive have successfully imported"), @ApiResponse(responseCode = "400", description = "Import process stuck in one of the steps:"
-                                                                                                                                                                    + "0 - Reading the archive content\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "1 - store information about the dossier template\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "2 - store the colors\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "3 - store the watermarks\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "4 - store the dossier status\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "5 - store the dossier attributes\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "6 - store the file attributes\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "7 - store the report templates\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "8 - store the legal basis\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "9 - store the file attribute configuration\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "10 - store the component definitions\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "11 - store the component mappings\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "12 - store the types and entities\n"
-                                                                                                                                                                    + "\n"
-                                                                                                                                                                    + "13 - store the rules and component rules"), @ApiResponse(responseCode = "404", description = "The dossier template to update does not exist")})
+    @Operation(summary = "Receives an archive to import", description = """
+            Import process stuck in one of the steps:"
+            0 - Reading the archive content
+            
+            1 - store information about the dossier template
+            
+            2 - store the colors
+            
+            3 - store the watermarks
+            
+            4 - store the dossier status
+            
+            5 - store the dossier attributes
+            
+            6 - store the file attributes
+            
+            7 - store the report templates
+            
+            8 - store the legal basis
+            
+            9 - store the file attribute configuration
+            
+            10 - store the component definitions
+            
+            11 - store the component mappings
+            
+            12 - store the types and entities
+            
+            13 - store the rules and component rules""")
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Archive have successfully imported"), @ApiResponse(responseCode = "400"), @ApiResponse(responseCode = "404", description = "The dossier template to update does not exist")})
     DossierTemplateModel importDossierTemplate(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                @RequestParam(value = DOSSIER_TEMPLATE_ID, required = false) String dossierTemplateId,
                                                @RequestParam(value = "updateExistingDossierTemplate", required = false, defaultValue = "false") boolean updateExistingDossierTemplate);

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ManualRedactionResource.java

@@ -119,8 +119,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse removeRedactionBulkLocal(@PathVariable(DOSSIER_ID) String dossierId,
                                                      @PathVariable(FILE_ID) String fileId,
-                                                     @RequestBody RemoveRedactionBulkLocalRequestModel removeRedactionRequest,
-                                                     @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                     @RequestBody RemoveRedactionBulkLocalRequestModel removeRedactionRequest);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -132,8 +131,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse removeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                 @PathVariable(FILE_ID) String fileId,
-                                                @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests,
-                                                @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                @RequestBody Set<RemoveRedactionRequestModel> removeRedactionRequests);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -170,8 +168,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse recategorizeBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                              @PathVariable(FILE_ID) String fileId,
-                                             @RequestBody Set<RecategorizationRequestModel> recategorizationRequests,
-                                             @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                             @RequestBody Set<RecategorizationRequestModel> recategorizationRequests);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -183,8 +180,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse recategorizeBulkLocal(@PathVariable(DOSSIER_ID) String dossierId,
                                                   @PathVariable(FILE_ID) String fileId,
-                                                  @RequestBody RecategorizationBulkLocalRequestModel recategorizationRequest,
-                                                  @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                  @RequestBody RecategorizationBulkLocalRequestModel recategorizationRequest);
 
 
     @ResponseStatus(value = HttpStatus.OK)
@@ -196,8 +192,7 @@ public interface ManualRedactionResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier  or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     ManualRedactionResponse resizeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                 @PathVariable(FILE_ID) String fileId,
-                                                @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests,
-                                                @RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
+                                                @RequestBody Set<ResizeRedactionRequestModel> resizeRedactionRequests);
 
 
     @ResponseStatus(value = HttpStatus.OK)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/StatusResource.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.external.resource;
 import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -16,6 +17,7 @@ import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.warning.ApproveResponse;
 
 import io.swagger.v3.oas.annotations.Operation;
@@ -25,6 +27,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
 public interface StatusResource {
 
     String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status";
+    String BY_ID_PATH = "/by-id";
     String CHANGES_SINCE_PATH = "/changes";
     String BULK_REST_PATH = "/bulk";
     String ASSIGNEE_REST_PATH = "/set-assignee";
@@ -56,6 +59,14 @@ public interface StatusResource {
     Map<String, List<FileStatus>> getDossierStatus(@RequestBody List<String> dossierIds);
 
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = STATUS_REST_PATH + BY_ID_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets the status for files by dossierId and fileIds.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    JSONPrimitive<Map<String, List<FileStatus>>> getFilesByIds(@RequestBody JSONPrimitive<Map<String, Set<String>>> filesByDossier);
+
+
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody
     @PostMapping(value = STATUS_REST_PATH + DELETED_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -146,9 +146,7 @@ public interface SupportResource {
 
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody
-    @PostMapping(value = DATASET_EXCHANGE
-                         + EXPORT
-                         + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = DATASET_EXCHANGE + EXPORT + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
             ## Export Preview Files Endpoint
             
@@ -166,6 +164,7 @@ public interface SupportResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
     ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
+
     @ResponseBody
     @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.service.v2.api.external.resource;
 import java.util.List;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DateFormatPatternErrorMessage;
 import com.iqser.red.service.persistence.service.v2.api.external.model.DossierAttributeDefinitionList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.DossierStatusDefinitionList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileAttributeDefinitionList;
@@ -63,6 +62,7 @@ public interface DossierTemplateResource {
     String DRY_RUN_PARAM = "dryRun";
     String ENCODING_PARAM = "encoding";
     String DELIMITER_PARAM = "delimiter";
+    String QUOTE_CHAR_PARAM = "quoteChar";
     String MAPPING_NAME_PARAM = "name";
 
     String INCLUDE_SOFT_DELETED = "includeSoftDeleted";
@@ -127,7 +127,7 @@ public interface DossierTemplateResource {
     @Operation(summary = "Upload a date formats file for a specific DossierTemplate.")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Date formats upload successful."), @ApiResponse(responseCode = "404", description = "The DossierTemplate is not found."), @ApiResponse(responseCode = "400", description = "Uploaded date formats could not be verified."), @ApiResponse(responseCode = "422", description = "Uploaded date formats file could not be parsed.")})
     ResponseEntity<?> uploadDateFormats(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
-                                                                          @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
+                                        @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
 
 
     @ResponseBody
@@ -159,7 +159,8 @@ public interface DossierTemplateResource {
                                                 @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                 @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
                                                 @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
-                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
+                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
+                                                @Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
 
 
     @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None")
@@ -173,7 +174,8 @@ public interface DossierTemplateResource {
                                                 @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
                                                 @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
                                                 @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
-                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
+                                                @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter,
+                                                @Parameter(name = QUOTE_CHAR_PARAM, description = "The quote char used in the file. Default is '\"'") @RequestParam(value = QUOTE_CHAR_PARAM, required = false, defaultValue = "\"") String quoteChar);
 
 
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml

@@ -509,7 +509,7 @@ paths:
         - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
         
         #### Customization Options
-        - Users can specify the delimiter and encoding used in the CSV file.
+        - Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
         
         #### Usage
         - The component mapping file can be utilized in component rules to relate components to existing master data.
@@ -533,6 +533,7 @@ paths:
         - $ref: '#/components/parameters/mappingName'
         - $ref: '#/components/parameters/encoding'
         - $ref: '#/components/parameters/delimiter'
+        - $ref: '#/components/parameters/quoteChar'
       responses:
         "200":
           content:
@@ -609,7 +610,7 @@ paths:
         - **Optimization Tip:** Place keys to be queried in the first columns and the results to be mapped in the last column for best performance.
         
         #### Customization Options
-        - Users can specify the delimiter and encoding used in the CSV file.
+        - Users can specify the delimiter, quoteChar, and encoding used in the CSV file.
       tags:
         - 1. Dossier Templates
       requestBody:
@@ -623,6 +624,7 @@ paths:
         - $ref: '#/components/parameters/mappingName'
         - $ref: '#/components/parameters/encoding'
         - $ref: '#/components/parameters/delimiter'
+        - $ref: '#/components/parameters/quoteChar'
       responses:
         "200":
           content:
@@ -2114,6 +2116,17 @@ components:
         example: ','
         default: ','
       description: "The delimiter used as a separator in a csv file."
+    quoteChar:
+      name: quoteChar
+      required: false
+      in: query
+      schema:
+        type: string
+        minLength: 1
+        maxLength: 1
+        example: '"'
+        default: '"'
+      description: "The quoteChar used to quote fields in a csv file."
     mappingName:
       name: name
       required: false

persistence-service-v1/persistence-service-processor-v1/build.gradle.kts

@@ -78,8 +78,8 @@ dependencies {
     api("com.opencsv:opencsv:5.9")
 
     implementation("com.google.protobuf:protobuf-java:4.27.1")
-    implementation("org.mapstruct:mapstruct:1.5.5.Final")
-    annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final")
+    implementation("org.mapstruct:mapstruct:1.6.2")
+    annotationProcessor("org.mapstruct:mapstruct-processor:1.6.2")
 
     testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
     testImplementation("org.testcontainers:postgresql:1.17.1")

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/RedObjectIdentityRetrievalStrategy.java

@@ -5,6 +5,7 @@ import org.springframework.security.acls.model.ObjectIdentity;
 import org.springframework.security.acls.model.ObjectIdentityRetrievalStrategy;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.IHavingDossierId;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 
 import lombok.extern.slf4j.Slf4j;
@@ -18,8 +19,8 @@ public class RedObjectIdentityRetrievalStrategy implements ObjectIdentityRetriev
         log.debug("Requesting data for object: {}", domainObject);
         if (domainObject instanceof Dossier) {
             return new ObjectIdentityImpl("Dossier", ((Dossier) domainObject).getId());
-        } else if (domainObject instanceof DossierChangeEntry) {
-            return new ObjectIdentityImpl("Dossier", ((DossierChangeEntry) domainObject).getDossierId());
+        } else if (domainObject instanceof IHavingDossierId) {
+            return new ObjectIdentityImpl("Dossier", ((IHavingDossierId) domainObject).getDossierId());
         } else if (domainObject instanceof String) {
             // TODO ACL this will not work once we have more than one type.
             return new ObjectIdentityImpl("Dossier", (String) domainObject);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/FileExchangeImportModel.java

@@ -50,7 +50,8 @@ public class FileExchangeImportModel {
         EntityLog entityLog;
         ComponentLog componentLog;
         List<ComponentLogEntry> overrides;
-        ManualChangesExportModel manualChanges;
+        @Builder.Default
+        ManualChangesExportModel manualChanges = new ManualChangesExportModel();
         @Builder.Default
         List<FileAttribute> fileAttributes = new ArrayList<>();
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualAddExportModel.java

@@ -11,13 +11,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualChangesExportModel.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.manualchanges;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import lombok.AccessLevel;
@@ -16,11 +17,17 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class ManualChangesExportModel {
 
-    List<ManualAddExportModel> manualAddExportModels;
-    List<ManualRecategorizationExportModel> manualRecategorizationExportModels;
-    List<ManualResizeExportModel> manualResizeExportModels;
-    List<ManualForceExportModel> manualForceExportModels;
-    List<ManualLegalBasisChangeExportModel> manualLegalBasisChangeExportModels;
-    List<ManualRemoveExportModel> manualRemoveExportModels;
+    @Builder.Default
+    List<ManualAddExportModel> manualAddExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualRecategorizationExportModel> manualRecategorizationExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualResizeExportModel> manualResizeExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualForceExportModel> manualForceExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualLegalBasisChangeExportModel> manualLegalBasisChangeExportModels = new ArrayList<>();
+    @Builder.Default
+    List<ManualRemoveExportModel> manualRemoveExportModels = new ArrayList<>();
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualForceExportModel.java

@@ -10,7 +10,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualLegalBasisChangeExportModel.java

@@ -8,7 +8,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class ManualLegalBasisChangeExportModel {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualRecategorizationExportModel.java

@@ -12,7 +12,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualRemoveExportModel.java

@@ -10,7 +10,6 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class ManualRemoveExportModel {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualResizeExportModel.java

@@ -14,7 +14,6 @@ import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierExportService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
 
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
@@ -9,6 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
@@ -33,29 +35,30 @@ public class DossierExportService {
     FileStatusManagementService fileStatusManagementService;
     EntityTypeExportService entityTypeExportService;
     ObjectMapper mapper;
+    FileStatusService fileStatusService;
 
 
     @SneakyThrows
     @Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
     public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
 
-        List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
+        List<String> fileIdsInDossier = fileStatusManagementService.getDossierStatusIds(dossier.getId(), false)
+                .stream()
+                .filter(fileId -> request.fileIds().isEmpty() || request.fileIds().contains(fileId))
+                .toList();
 
-        if (!request.dossierIds().contains(dossier.getId()) //
-            && files.stream()
-                    .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
-            // dossier has no files in requested files and dossier not explicitly requested -> don't export it.
+        if (fileIdsInDossier.isEmpty()) {
             return;
         }
 
         Path dossierFolder = folder.resolve(dossier.getId());
         archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
 
-        for (FileModel fileEntity : files) {
-            if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
-                continue;
-            }
-            fileExportService.addFileToArchive(archiver, dossierFolder, request, fileEntity);
+        for (String fileId : fileIdsInDossier) {
+
+            FileModel file = fileStatusService.getStatus(fileId);
+
+            fileExportService.addFileToArchive(archiver, dossierFolder, request, file);
         }
 
         List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateImportService.java

@@ -41,6 +41,7 @@ import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zi
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DateFormatsValidationService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DefaultDateFormatsProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReportTemplateService;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
@@ -75,10 +76,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -109,6 +108,7 @@ public class DossierTemplateImportService {
     private final ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
     private final EntityTypeImportService entityTypeImportService;
     private final SystemManagedTypesImport systemManagedTypesImport;
+    private final DefaultDateFormatsProvider defaultDateFormatsProvider;
 
 
     public String importDossierTemplate(ImportDossierTemplateRequest request) {
@@ -449,7 +449,8 @@ public class DossierTemplateImportService {
                                                                                      componentMapping.metadata().getFileName(),
                                                                                      componentMapping.metadata().getDelimiter(),
                                                                                      componentMapping.metadata().getEncoding(),
-                                                                                     tmpFile);
+                                                                                     tmpFile,
+                                                                                     componentMapping.metadata().getQuoteChar());
 
             componentMappingService.setVersion(createdMapping.getId(), componentMapping.metadata().getVersion());
             assert tmpFile.delete();
@@ -494,13 +495,18 @@ public class DossierTemplateImportService {
 
     private void setDataFormats(ImportTemplateResult request, String dossierTemplateId) {
 
-        if (request.getDateFormats() != null) {
-            List<DateFormatPatternErrorMessage> dateFormatPatternErrorMessages = dateFormatsValidationService.validateDateFormats(request.getDateFormats());
+        String dateFormats = request.getDateFormats();
+        if (dateFormats == null && applicationType.equals("DocuMine")) {
+            dateFormats = defaultDateFormatsProvider.getDateFormats();
+        }
+        if (dateFormats != null) {
+            List<DateFormatPatternErrorMessage> dateFormatPatternErrorMessages = dateFormatsValidationService.validateDateFormats(dateFormats);
             if (!dateFormatPatternErrorMessages.isEmpty()) {
                 throw new BadRequestException("The date formats file contains errors");
             }
-            dateFormatsPersistenceService.setDateFormats(request.dateFormats, dossierTemplateId, request.getDateFormatsExportModel().getVersion());
+            dateFormatsPersistenceService.setDateFormats(dateFormats, dossierTemplateId, request.getDateFormatsExportModel().getVersion());
         }
+
     }
 
 
@@ -524,6 +530,7 @@ public class DossierTemplateImportService {
 
     }
 
+
     private LayoutParsingType deferFromApplicationType() {
 
         return Objects.equals(applicationType, "DocuMine") ? LayoutParsingType.DOCUMINE_OLD : LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH;
@@ -627,6 +634,9 @@ public class DossierTemplateImportService {
 
     private void updateComponents(ImportTemplateResult request, String dossierTemplateId) {
 
+        List<ComponentDefinitionEntity> existingComponentDefinitions = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplateId);
+        existingComponentDefinitions.forEach(componentDefinition -> componentDefinitionPersistenceService.delete(componentDefinition.getId()));
+
         request.getComponentDefinitions()
                 .forEach(componentDefinition -> {
                     componentDefinition.setDossierTemplateId(dossierTemplateId);
@@ -686,12 +696,12 @@ public class DossierTemplateImportService {
 
             importModel.getTypes().removeIf(t -> systemManagedTypesIdsFromImport.contains(t.getType()));
 
-            importModel.getEntries().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getFalsePositives().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getFalseRecommendations().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getDeletedEntries().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getDeletedFalsePositives().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
-            importModel.getDeletedFalseRecommendations().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
+            importModel.getEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+            importModel.getFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+            importModel.getFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+            importModel.getDeletedEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+            importModel.getDeletedFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+            importModel.getDeletedFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
         }
         return importModel;
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java

@@ -3,11 +3,14 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
@@ -83,7 +86,7 @@ public class FileExchangeExportService {
 
 
     @Observed(name = "FileExchangeExportService", contextualName = "export-files")
-    public long createDownloadArchive(ExportDownloadMessage downloadJob) throws IOException {
+    public long createDownloadArchive(ExportDownloadMessage downloadJob) {
 
         FileExchangeExportRequest request = downloadJob.getFileExchangeExportRequest();
         DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(downloadJob.getDossierTemplateId());
@@ -92,13 +95,18 @@ public class FileExchangeExportService {
 
             dossierTemplateExportService.addDossierTemplateToArchive(archiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate);
 
-            for (Dossier dossierEntity : dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplate.getId(), true, false)) {
+            Iterable<String> dossierIds;
+            if (request.dossierIds().isEmpty()) {
+                dossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplate.getId(), true, false);
+            } else {
+                dossierIds = request.dossierIds();
+            }
 
-                if (!request.dossierIds().isEmpty() && !request.dossierIds().contains(dossierEntity.getId())) {
-                    continue;
-                }
+            for (String dossierId : dossierIds) {
 
-                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
+                Dossier dossier = dossierManagementService.getDossierById(dossierId, true, false);
+
+                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossier);
             }
 
             storeZipFile(downloadJob.getStorageId(), archiver);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExportService.java

@@ -29,7 +29,6 @@ import lombok.extern.slf4j.Slf4j;
 public class FileExportService {
 
     ManualChangesExportService manualChangesExportService;
-    ComponentLogService componentLogService;
     ObjectMapper mapper;
     FileManagementStorageService storageService;
 
@@ -41,10 +40,12 @@ public class FileExportService {
         Path fileFolder = folder.resolve(file.getId());
 
         if (!request.excludeAnalysisLogs()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
-                                                                        mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
-            if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
+            if (storageService.entityLogExists(file.getDossierId(), file.getId())) {
+                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+                                                                            buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
+                                                                            mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
+            }
+            if (storageService.componentLogExists(file.getDossierId(), file.getId())) {
                 archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
                                                                             buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
                                                                             mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/ComponentMappingEntity.java

@@ -75,4 +75,7 @@ public class ComponentMappingEntity {
     @Builder.Default
     char delimiter = ',';
 
+    @Builder.Default
+    char quoteChar = '"';
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/AnnotationEntityId.java

@@ -12,7 +12,6 @@ import lombok.NoArgsConstructor;
 
 @Embeddable
 @Data
-@Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class AnnotationEntityId implements Serializable {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java

@@ -17,14 +17,12 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Entity
 @Table(name = "id_removal")
 @Data
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 public class IdRemovalEntity implements IBaseAnnotation {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java

@@ -15,7 +15,7 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
+//@Builder // disabled, cause the ManualChangesExportMapper will not map addToDictionary and addToAllDossiers at all
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
@@ -36,8 +36,6 @@ public class ManualForceRedactionEntity implements IBaseAnnotation {
     private OffsetDateTime softDeletedTime;
     @Column
     private int page;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java

@@ -10,12 +10,10 @@ import jakarta.persistence.Entity;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
@@ -41,8 +39,6 @@ public class ManualLegalBasisChangeEntity implements IBaseAnnotation {
     private OffsetDateTime softDeletedTime;
     @Column
     private int page;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRecategorizationEntity.java

@@ -17,12 +17,10 @@ import jakarta.persistence.Entity;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
@@ -56,8 +54,6 @@ public class ManualRecategorizationEntity implements IBaseAnnotation {
     private String section;
     @Column
     private String value;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java

@@ -22,13 +22,11 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
 @Entity
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Table(name = "manual_redaction")
@@ -51,13 +49,13 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation {
     @Column
     private boolean rectangle;
     @Column
-    private boolean addToDictionary;
+    private boolean addToDictionary; //true, true
     @Column
-    private boolean addToAllDossiers;
+    private boolean addToAllDossiers; //true, false
 
     @Column
     @Deprecated
-    private boolean addToDossierDictionary;
+    private boolean addToDossierDictionary;  //false, true
 
     @Column
     @Enumerated(EnumType.STRING)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java

@@ -19,12 +19,10 @@ import jakarta.persistence.FetchType;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
@@ -63,9 +61,6 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation {
     @Column
     private boolean addToAllDossiers;
 
-    @Column
-    private String basedOnDictAnnotationId;
-
     @ElementCollection
     @Fetch(value = FetchMode.SUBSELECT)
     private Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ApplicationConfigurationEntity.java

@@ -24,15 +24,19 @@ public class ApplicationConfigurationEntity {
     private final String id = ApplicationConfigurationEntity.ID;
 
     @Column
+    @Builder.Default
     private int downloadCleanupDownloadFilesHours = 8;
 
     @Column
+    @Builder.Default
     private int downloadCleanupNotDownloadFilesHours = 72;
 
     @Column
+    @Builder.Default
     private int softDeleteCleanupTime = 96;
 
     @Column
+    @Builder.Default
     private int hardDeleteCleanupRetryTime = 72;
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java

@@ -10,6 +10,7 @@ import org.hibernate.annotations.Fetch;
 import org.hibernate.annotations.FetchMode;
 
 import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
 
@@ -201,6 +202,11 @@ public class FileEntity {
     @Column
     private OffsetDateTime errorTimestamp;
 
+    @Column
+    @Enumerated(EnumType.STRING)
+    private ErrorCode errorCode;
+
+
     @ElementCollection(fetch = FetchType.EAGER)
     private List<FileEntityComponentMappingVersionEntity> componentMappingVersions;
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ManualChangesExportMapper.java

@@ -73,65 +73,76 @@ public interface ManualChangesExportMapper {
 
     @AfterMapping
     default void setFileStatusAndId(ManualLegalBasisChangeExportModel model,
-                                    @MappingTarget ManualLegalBasisChangeEntity.ManualLegalBasisChangeEntityBuilder entity,
-                                    @Context FileEntity file, @Context String userId) {
+                                    @MappingTarget ManualLegalBasisChangeEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
     default void setFileStatusAndId(ManualRecategorizationExportModel model,
-                                    @MappingTarget ManualRecategorizationEntity.ManualRecategorizationEntityBuilder entity,
-                                    @Context FileEntity file, @Context String userId) {
+                                    @MappingTarget ManualRecategorizationEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualAddExportModel model, @MappingTarget ManualRedactionEntryEntity.ManualRedactionEntryEntityBuilder entity, @Context FileEntity file, @Context String userId) {
+    default void setFileStatusAndId(ManualAddExportModel model,
+                                    @MappingTarget ManualRedactionEntryEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualResizeExportModel model, @MappingTarget ManualResizeRedactionEntity.ManualResizeRedactionEntityBuilder entity, @Context FileEntity file, @Context String userId) {
+    default void setFileStatusAndId(ManualResizeExportModel model,
+                                    @MappingTarget ManualResizeRedactionEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualForceExportModel model, @MappingTarget ManualForceRedactionEntity.ManualForceRedactionEntityBuilder entity, @Context FileEntity file, @Context String userId) {
+    default void setFileStatusAndId(ManualForceExportModel model,
+                                    @MappingTarget ManualForceRedactionEntity entity,
+                                    @Context FileEntity file,
+                                    @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 
     @AfterMapping
-    default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity.IdRemovalEntityBuilder entity, @Context FileEntity file, @Context String userId) {
+    default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity entity, @Context FileEntity file, @Context String userId) {
 
         AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
-        entity.id(annotationEntityId);
-        entity.fileStatus(file);
-        entity.user(userId);
+        entity.setId(annotationEntityId);
+        entity.setFileStatus(file);
+        entity.setUser(userId);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.migration;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 
 import org.springframework.stereotype.Service;
 
@@ -12,8 +13,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
 import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 
 import lombok.RequiredArgsConstructor;
@@ -39,26 +38,27 @@ public class SaasMigrationManualChangesUpdateService {
             if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) {
                 // copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table.
                 // It has no functionality, its only there, such that the UI can show a pending change.
-                ManualRedactionEntryEntity pendingDictAdd = ManualRedactionEntryEntity.builder()
-                        .id(buildSecondaryId(unprocessedManualAdd.getId(), fileId))
-                        .user(unprocessedManualAdd.getUser())
-                        .typeId(unprocessedManualAdd.getTypeId())
-                        .value(unprocessedManualAdd.getValue())
-                        .reason(unprocessedManualAdd.getReason())
-                        .legalBasis(unprocessedManualAdd.getLegalBasis())
-                        .section(unprocessedManualAdd.getSection())
-                        .rectangle(unprocessedManualAdd.isRectangle())
-                        .addToDictionary(unprocessedManualAdd.isAddToDictionary())
-                        .addToAllDossiers(unprocessedManualAdd.isAddToAllDossiers())
-                        .dictionaryEntryType(DictionaryEntryType.ENTRY)
-                        .requestDate(unprocessedManualAdd.getRequestDate())
-                        .positions(new ArrayList<>(unprocessedManualAdd.getPositions())) // copy to new List
-                        .fileStatus(unprocessedManualAdd.getFileStatus())
-                        .textBefore(unprocessedManualAdd.getTextBefore())
-                        .textAfter(unprocessedManualAdd.getTextAfter())
-                        .sourceId(unprocessedManualAdd.getSourceId())
-                        .typeIdsOfModifiedDictionaries(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries())
-                        .build();
+                ManualRedactionEntryEntity pendingDictAdd = new ManualRedactionEntryEntity(buildSecondaryId(unprocessedManualAdd.getId(), fileId),
+                                                                                           unprocessedManualAdd.getUser(),
+                                                                                           unprocessedManualAdd.getTypeId(),
+                                                                                           unprocessedManualAdd.getValue(),
+                                                                                           unprocessedManualAdd.getReason(),
+                                                                                           unprocessedManualAdd.getLegalBasis(),
+                                                                                           unprocessedManualAdd.getSection(),
+                                                                                           unprocessedManualAdd.isRectangle(),
+                                                                                           unprocessedManualAdd.isAddToDictionary(),
+                                                                                           unprocessedManualAdd.isAddToAllDossiers(),
+                                                                                           unprocessedManualAdd.isAddToDossierDictionary(),
+                                                                                           DictionaryEntryType.ENTRY,
+                                                                                           unprocessedManualAdd.getRequestDate(),
+                                                                                           null,
+                                                                                           null,
+                                                                                           new ArrayList<>(unprocessedManualAdd.getPositions()),
+                                                                                           unprocessedManualAdd.getFileStatus(),
+                                                                                           unprocessedManualAdd.getTextBefore(),
+                                                                                           unprocessedManualAdd.getTextAfter(),
+                                                                                           unprocessedManualAdd.getSourceId(),
+                                                                                           new HashSet<>(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries()));
 
                 addRedactionPersistenceService.update(pendingDictAdd);
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java

@@ -380,7 +380,7 @@ public class SaasMigrationService implements TenantSyncService {
 
     private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
 
-        return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build();
+        return new AnnotationEntityId(annotationId, fileId);
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/AddDateFormatsToTemplatesMigration25.java

@@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.DefaultDateFormatsProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 
@@ -33,11 +34,13 @@ public class AddDateFormatsToTemplatesMigration25 extends Migration {
     DateFormatsPersistenceService dateFormatsPersistenceService;
     @Autowired
     DossierTemplatePersistenceService dossierTemplatePersistenceService;
-
+    @Autowired
+    DefaultDateFormatsProvider defaultDateFormatsProvider;
 
     @Value("${application.type}")
     private String applicationType;
 
+
     public AddDateFormatsToTemplatesMigration25() {
 
         super(NAME, VERSION);
@@ -48,19 +51,15 @@ public class AddDateFormatsToTemplatesMigration25 extends Migration {
     @SneakyThrows
     protected void migrate() {
 
-        if(!applicationType.equalsIgnoreCase("DocuMine")){
+        if (!applicationType.equalsIgnoreCase("DocuMine")) {
             log.info("Skipping AddDateFormatsToTemplatesMigration25 as application type is not DocuMine!!!");
             return;
         }
 
         List<DossierTemplateEntity> allDossierTemplates = dossierTemplatePersistenceService.getAllDossierTemplates();
         allDossierTemplates.forEach(dt -> {
-            Resource resource = new ClassPathResource("files/dateFormats.txt");
-
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) {
-                String dateFormats = reader.lines().collect(Collectors.joining(System.lineSeparator()));
-                dateFormatsPersistenceService.setDateFormats(dateFormats, dt.getId(), -1);
-
+            try {
+                dateFormatsPersistenceService.setDateFormats(defaultDateFormatsProvider.getDateFormats(), dt.getId(), -1);
             } catch (Exception e) {
                 log.info("Could not update dossier template {}, error: {}", dt.getId(), e.getMessage());
             }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualChangesReorderingMigration23.java

@@ -0,0 +1,90 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import java.time.OffsetDateTime;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.projections.EntryWithManualChangesProjection;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class ManualChangesReorderingMigration23 extends Migration {
+
+    private static final String NAME = "Migration for reordering mixed up manual changes";
+    private static final long VERSION = 23;
+    private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
+    private final AddRedactionPersistenceService addRedactionPersistenceService;
+    private final FileStatusService fileStatusService;
+
+
+    public ManualChangesReorderingMigration23(EntityLogEntryDocumentRepository entityLogEntryDocumentRepository,
+                                              AddRedactionPersistenceService addRedactionPersistenceService,
+                                              FileStatusService fileStatusService) {
+
+        super(NAME, VERSION);
+        this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;
+        this.addRedactionPersistenceService = addRedactionPersistenceService;
+        this.fileStatusService = fileStatusService;
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Searching for entityLogEntries of type ENTITY with state APPLIED and no legalbasis");
+        List<EntryWithManualChangesProjection> entriesWithManualChanges = entityLogEntryDocumentRepository.findAppliedEntitiesWhereLegalBasisEmpty();
+        log.info("Found {} applied entries with no legal basis", entriesWithManualChanges.size());
+        int numberOfChanged = 0;
+        Set<FileAndDossier> affectedFiles = new HashSet<>();
+        for (EntryWithManualChangesProjection entry : entriesWithManualChanges) {
+            if (entry.getManualChanges().isEmpty()) {
+                continue;
+            }
+            Optional<ManualChange> optionalAdd = entry.getManualChanges()
+                    .stream()
+                    .filter(mc -> mc.getManualRedactionType().equals(ManualRedactionType.ADD))
+                    .findFirst();
+            if (optionalAdd.isEmpty()) {
+                continue;
+            }
+            ManualChange add = optionalAdd.get();
+            ManualChange first = entry.getManualChanges()
+                    .get(0);
+            if (add.equals(first)) {
+                continue;
+            }
+            OffsetDateTime firstTimestamp = first.getRequestedDate().minusSeconds(1);
+            String dossierId = entry.getEntityLogId().split("/")[0];
+            String fileId = entry.getEntityLogId().split("/")[1];
+            int i = addRedactionPersistenceService.updateRequestDate(fileId, entry.getEntryId(), firstTimestamp);
+            if (i != 0) {
+                affectedFiles.add(new FileAndDossier(fileId, dossierId));
+                numberOfChanged += i;
+            }
+        }
+        log.info("Updated request date of {} manual add redactions in {} files.", numberOfChanged, affectedFiles.size());
+        log.info("Reanalyzing affected files");
+        for (FileAndDossier affectedFile : affectedFiles) {
+            fileStatusService.setStatusReprocess(affectedFile.dossierId(), affectedFile.fileId(), false);
+        }
+
+    }
+
+
+    private record FileAndDossier(String fileId, String dossierId) {
+
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
 
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -14,7 +13,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileMan
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/DossierIdFilterable.java

@@ -0,0 +1,6 @@
+package com.iqser.red.service.persistence.management.v1.processor.model;
+
+public interface DossierIdFilterable {
+
+    String getDossierId();
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java

@@ -62,16 +62,23 @@ public class ComponentMappingService {
 
 
     @SneakyThrows
-    public ComponentMappingMetadata update(String dossierTemplateId, String mappingId, String name, String encoding, char delimiter, File mappingFile, String fileName) {
+    public ComponentMappingMetadata update(String dossierTemplateId,
+                                           String mappingId,
+                                           String name,
+                                           String encoding,
+                                           char delimiter,
+                                           File mappingFile,
+                                           String fileName,
+                                           char quoteChar) {
 
         ComponentMappingEntity entity = componentMappingPersistenceService.getEntityById(dossierTemplateId, mappingId);
 
-        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName);
+        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName, quoteChar);
     }
 
 
     @SneakyThrows
-    public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile) {
+    public ComponentMappingMetadata create(String dossierTemplateId, String name, String fileName, char delimiter, String encoding, File mappingFile, char quoteChar) {
 
         if (componentMappingPersistenceService.existsByNameAndDossierTemplateId(name, dossierTemplateId)) {
             throw new BadRequestException("A mapping with this name already exists in the dossier template!");
@@ -86,20 +93,27 @@ public class ComponentMappingService {
                 .fileName(fileName)
                 .build();
 
-        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName);
+        return updateOrCreate(entity, name, encoding, delimiter, mappingFile, fileName, quoteChar);
 
     }
 
 
     @SneakyThrows
-    private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity, String name, String encoding, char delimiter, File mappingFile, String fileName) {
+    private ComponentMappingMetadata updateOrCreate(ComponentMappingEntity entity,
+                                                    String name,
+                                                    String encoding,
+                                                    char delimiter,
+                                                    File mappingFile,
+                                                    String fileName,
+                                                    char quoteChar) {
 
         Charset charset = resolveCharset(encoding);
 
-        CsvStats stats = sortCSVFile(delimiter, mappingFile, charset);
+        CsvStats stats = sortCSVFile(delimiter, mappingFile, charset, quoteChar);
 
         entity.setName(name);
         entity.setDelimiter(delimiter);
+        entity.setQuoteChar(quoteChar);
         entity.setEncoding(encoding);
         entity.setNumberOfLines(stats.numberOfLines());
         entity.setColumnLabels(stats.columnLabels());
@@ -126,7 +140,7 @@ public class ComponentMappingService {
     }
 
 
-    private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws BadRequestException, IOException {
+    private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset, char quoteChar) throws BadRequestException, IOException {
 
         Path tempFile = Files.createTempFile("mapping", ".tmp");
 
@@ -135,11 +149,8 @@ public class ComponentMappingService {
         String[] columnLabels;
         int numberOfLines = 0;
         try (Reader fileReader = new FileReader(tempFile.toFile(), charset);//
-             CSVReader reader = buildReader(fileReader, delimiter);//
-             CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset), delimiter,
-                                              CSVWriter.NO_QUOTE_CHARACTER,
-                                              CSVWriter.DEFAULT_ESCAPE_CHARACTER,
-                                              CSVWriter.DEFAULT_LINE_END)) {
+             CSVReader reader = buildReader(fileReader, delimiter, quoteChar);//
+             CSVWriter writer = new CSVWriter(new FileWriter(mappingFile, charset), delimiter, quoteChar, '\\', CSVWriter.DEFAULT_LINE_END)) {
 
             List<String[]> rows = reader.readAll();
 
@@ -180,9 +191,9 @@ public class ComponentMappingService {
     }
 
 
-    private static CSVReader buildReader(Reader reader, char delimiter) throws IOException {
+    private static CSVReader buildReader(Reader reader, char delimiter, char quoteChar) throws IOException {
 
-        return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build();
+        return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).withQuoteChar(quoteChar).build()).build();
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DefaultDateFormatsProvider.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.persistence.management.v1.processor.service;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.stream.Collectors;
+
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.stereotype.Service;
+
+import lombok.SneakyThrows;
+
+@Service
+public class DefaultDateFormatsProvider {
+
+    private String dateFormats;
+
+
+    @SneakyThrows
+    public String getDateFormats() {
+
+        if (dateFormats == null) {
+            Resource resource = new ClassPathResource("files/dateFormats.txt");
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) {
+                dateFormats = reader.lines()
+                        .collect(Collectors.joining(System.lineSeparator()));
+            }
+        }
+
+        return dateFormats;
+    }
+
+}
+

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java

@@ -189,8 +189,6 @@ public class DictionaryService {
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DOSSIER_DICTIONARY_TYPE + "')")
     public void updateDossierType(String type, String dossierTemplateId, UpdateTypeValue typeValue, String dossierId, TypeEntity typeEntity) {
 
-        accessControlService.verifyUserIsDossierOwner(dossierId);
-
         if (typeEntity.isDossierDictionaryOnly()) {
             dictionaryManagementService.updateTypeValue(toTypeId(type, dossierTemplateId, dossierId),
                                                         Type.builder()

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java

@@ -23,11 +23,12 @@ public class DossierIdFileIdRequestValidator {
     DossierManagementService dossierManagementService;
     FileStatusManagementService fileStatusManagementService;
 
+
     @Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request")
     public void validateRequestOrThrow404(String dossierTemplateId, Set<String> dossierIds, Set<String> fileIds) {
 
         if (!dossierIds.isEmpty()) {
-            Set<String> availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
+            Set<String> availableDossierIds = new HashSet<>(dossierManagementService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, dossierIds));
             Set<String> nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds);
             if (!nonAvailableDossiers.isEmpty()) {
                 throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", String.join(", ", nonAvailableDossiers), dossierTemplateId));
@@ -35,9 +36,19 @@ public class DossierIdFileIdRequestValidator {
         }
 
         if (!fileIds.isEmpty()) {
-            Set<String> availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplateId)
+            Set<FileModel> availableFiles = fileStatusManagementService.findAllByIds(fileIds)
                     .stream()
                     .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
+                    .collect(Collectors.toSet());
+
+            Set<String> mentionedDossierIds = availableFiles.stream()
+                    .map(FileModel::getDossierId)
+                    .collect(Collectors.toSet());
+
+            Set<String> availableDossierIds = new HashSet<>(dossierManagementService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, mentionedDossierIds));
+
+            Set<String> availableFileIds = availableFiles.stream()
+                    .filter(file -> availableDossierIds.contains(file.getDossierId()))
                     .map(FileModel::getId)
                     .collect(Collectors.toSet());
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.utils.DossierMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierInformation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
@@ -120,9 +121,15 @@ public class DossierManagementService {
     }
 
     @Transactional
-    public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
 
-        return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
+        return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
+    @Transactional
+    public List<String> findAllDossierIdsInDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
+
+        return dossierService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, dossierIds);
 
     }
 
@@ -269,4 +276,16 @@ public class DossierManagementService {
 
     }
 
+
+    public DossierChangeResponseV2 changesSinceV2(JSONPrimitive<OffsetDateTime> since) {
+        return dossierService.changesSinceV2(since.getValue());
+    }
+
+
+    @Transactional
+    public List<Dossier> getDossiersByIds(Set<String> viewableDossierIds) {
+
+        return getConvertedAllDossiers(dossierService.getAllDossiers(viewableDossierIds), true,true);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
 import java.time.OffsetDateTime;
+import java.util.Collection;
 import java.util.List;
 import java.util.Set;
 
@@ -14,6 +15,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierChange;
@@ -147,7 +149,7 @@ public class DossierService {
     }
 
 
-    public List<DossierEntity> getAllDossiers(List<String> dossierIds) {
+    public List<DossierEntity> getAllDossiers(Collection<String> dossierIds) {
 
         return dossierPersistenceService.findAllDossiers(dossierIds);
     }
@@ -159,12 +161,17 @@ public class DossierService {
     }
 
 
-    public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> findAllDossierIdsInDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
 
-        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId);
+        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId, dossierIds);
     }
 
 
+    public List<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
+
+        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
     public Set<DossierChange> changesSince(OffsetDateTime since) {
 
         return dossierPersistenceService.hasChangesSince(since);
@@ -182,4 +189,9 @@ public class DossierService {
         dossierPersistenceService.unarchiveDossier(dossierId);
     }
 
+
+    public DossierChangeResponseV2 changesSinceV2(OffsetDateTime value) {
+        return dossierPersistenceService.hasChangesSinceV2(value);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java

@@ -216,7 +216,8 @@ public class DossierTemplateCloneService {
                                            componentMapping.metaData().getFileName(),
                                            componentMapping.metaData().getDelimiter(),
                                            componentMapping.metaData().getEncoding(),
-                                           componentMapping.file());
+                                           componentMapping.file(),
+                                           componentMapping.metaData().getQuoteChar());
         }
         FileSystemUtils.deleteRecursively(dir);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java

@@ -103,13 +103,17 @@ public class EntityLogMergeService {
                                        int analysisNumber,
                                        Map<String, List<BaseAnnotation>> allManualChanges) {
 
-        Map<String, String> trackLocalChangesBasedOnDictEntriesMap = new HashMap<>();
         List<EntityLogEntry> mergedEntityLogEntries = new LinkedList<>(entityLogEntries);
         Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber)//
                 .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
         mergedEntityLogEntries.addAll(addedLocalManualEntries.values());
         buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(mergedEntityLogEntries::add);
 
+        Map<String, String> trackLocalChangesBasedOnDictEntriesMap = unprocessedManualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(ManualRedactionEntry::isLocal)
+                .filter(entry -> entry.getSourceId() != null && !entry.getSourceId().isEmpty())
+                .collect(Collectors.toMap(ManualRedactionEntry::getAnnotationId, ManualRedactionEntry::getSourceId));
         processEntityLogEntries(dossier, mergedEntityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges, trackLocalChangesBasedOnDictEntriesMap);
 
         adjustEntityLogEntriesAfterLocalChangesBasedOnDict(entityLogEntries, trackLocalChangesBasedOnDictEntriesMap, analysisNumber);
@@ -121,7 +125,7 @@ public class EntityLogMergeService {
                                                                     Map<String, String> trackLocalChangesBasedOnDictEntriesMap,
                                                                     int analysisNumber) {
 
-        var dictEntryIdsToUpdate = trackLocalChangesBasedOnDictEntriesMap.values();
+        Set<String> dictEntryIdsToUpdate = new HashSet<>(trackLocalChangesBasedOnDictEntriesMap.values());
         entityLogEntries.stream()
                 .filter(entityLogEntry -> dictEntryIdsToUpdate.contains(entityLogEntry.getId()))
                 .forEach(entityLogEntry -> {
@@ -268,26 +272,14 @@ public class EntityLogMergeService {
                         return null;
                     } else if (localChange instanceof ManualResizeRedaction manualResizeRedaction) {
                         mergeResizeRedaction(manualResizeRedaction, entityLogEntry, analysisNumber);
-                        if (manualResizeRedaction.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualResizeRedaction.getAnnotationId(), manualResizeRedaction.getBasedOnDictAnnotationId());
-                        }
 
                         return null;
                     } else if (localChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
                         mergeLegalBasisChange(manualLegalBasisChange, entityLogEntry, analysisNumber);
-                        if (manualLegalBasisChange.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualLegalBasisChange.getAnnotationId(), manualLegalBasisChange.getBasedOnDictAnnotationId());
-                        }
                         return null;
                     } else if (localChange instanceof ManualRecategorization manualRecategorization) {
-                        if (manualRecategorization.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualRecategorization.getAnnotationId(), manualRecategorization.getBasedOnDictAnnotationId());
-                        }
                         return mergeRecategorization(manualRecategorization, entityLogEntry, dossier, analysisNumber);
                     } else if (localChange instanceof ManualForceRedaction manualForceRedaction) {
-                        if (manualForceRedaction.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualForceRedaction.getAnnotationId(), manualForceRedaction.getBasedOnDictAnnotationId());
-                        }
                         mergeForceRedaction(manualForceRedaction, entityLogEntry, analysisNumber);
                         return null;
                     } else {
@@ -393,11 +385,10 @@ public class EntityLogMergeService {
 
         entityLogEntry.setState(EntryState.IGNORED);
         //special case, only for add local and remove only
-        if (entityLogEntry.getEngines().equals(Set.of(Engine.MANUAL))) {
+        if (entityLogEntry.getEngines().contains(Engine.MANUAL)) {
             entityLogEntry.setState(EntryState.REMOVED);
             change.setType(ChangeType.REMOVED);
         }
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(idRemoval, 0));
 
         changes.add(change);
@@ -435,7 +426,6 @@ public class EntityLogMergeService {
         entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
         entityLogEntry.setPositions(newPositions);
         entityLogEntry.setValue(manualResizeRedaction.getValue());
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualResizeRedaction, 0));
     }
 
@@ -464,7 +454,6 @@ public class EntityLogMergeService {
         entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
         entityLogEntry.setSection(manualLegalBasisChange.getSection());
         entityLogEntry.setValue(manualLegalBasisChange.getValue());
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualLegalBasisChange, 0));
     }
 
@@ -486,8 +475,6 @@ public class EntityLogMergeService {
             return pendingEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
         }
 
-        entityLogEntry.getEngines().add(Engine.MANUAL);
-
         if (recategorization.getType() != null && !recategorization.getType().equals(entityLogEntry.getType())) {
             boolean isHint = isHint(recategorization.getType(), dossier);
             entityLogEntry.setType(recategorization.getType());
@@ -549,10 +536,6 @@ public class EntityLogMergeService {
                                            PropertyChange.builder().property("state").oldValue(oldState.name()).newValue(newState.name()).build()));
         entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
         entityLogEntry.setState(newState);
-        entityLogEntry.getEngines().add(Engine.MANUAL);
-        if (forceRedaction.getBasedOnDictAnnotationId() != null) {
-            entityLogEntry.getEngines().add(Engine.DICTIONARY);
-        }
         addChanges(entityLogEntry, changes);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(forceRedaction, 0));
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java

@@ -170,6 +170,11 @@ public class FileManagementStorageService {
         return entityLogMongoService.entityLogDocumentExists(dossierId, fileId);
     }
 
+    public boolean componentLogExists(String dossierId, String fileId) {
+
+        return componentLogMongoService.componentLogDocumentExists(dossierId, fileId);
+    }
+
 
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 
@@ -246,6 +251,8 @@ public class FileManagementStorageService {
 
     public void deleteAllObjects(String dossierId, String fileId) {
 
+        deleteObject(dossierId, fileId, FileType.VIEWER_DOCUMENT);
+
         deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
         deleteEntityLog(dossierId, fileId);
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusManagementService.java

@@ -23,7 +23,6 @@ public class FileStatusManagementService {
 
     private final FileStatusService fileStatusService;
     private final ExcludeFromAnalysisService excludeFromAnalysis;
-    private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
     private final IndexingService indexingService;
 
 
@@ -38,11 +37,13 @@ public class FileStatusManagementService {
         return fileStatusService.getAllFiles();
     }
 
+
     public List<FileModel> getAllDossierTemplateStatus(String dossierTemplateId) {
 
         return fileStatusService.getDossierTemplateStatus(dossierTemplateId);
     }
 
+
     public List<FileModel> getDossierStatus(String dossierId) {
 
         return fileStatusService.getDossierStatus(dossierId)
@@ -51,6 +52,16 @@ public class FileStatusManagementService {
                 .collect(Collectors.toList());
     }
 
+    public List<FileModel> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) {
+
+        return fileStatusService.findAllDossierIdAndIds(dossierId,fileIds);
+    }
+
+
+    public List<String> getDossierStatusIds(String dossierId, boolean includeDeleted) {
+
+        return fileStatusService.getDossierStatusIds(dossierId, includeDeleted);
+    }
 
 
     public List<FileModel> getAllDossierStatus(String dossierId) {
@@ -194,4 +205,10 @@ public class FileStatusManagementService {
         fileStatusService.setExcludedPages(fileId, excludedPages);
     }
 
+
+    public List<FileModel> findAllByIds(Set<String> fileIds) {
+
+        return fileStatusService.findAllByIds(fileIds);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java

@@ -8,11 +8,10 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.function.BiFunction;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
 import org.apache.commons.lang3.StringUtils;
+
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
@@ -24,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.configuration.M
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ComponentDefinitionEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@@ -144,6 +144,7 @@ public class FileStatusService {
         return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
     }
 
+
     public List<DossierStatsFileProjection> getDossierStatsFiles(String dossierId) {
 
         return fileStatusPersistenceService.getFilesForDossierStats(dossierId);
@@ -1066,4 +1067,31 @@ public class FileStatusService {
         addSearchTermOccurrencesAnalysisRequestToAnalysisQueue(fileStatus, bulkLocalRequest);
     }
 
+
+    public List<FileModel> findAllByIds(Set<String> fileIds) {
+
+        return fileStatusPersistenceService.findAllByIds(fileIds)
+                .stream()
+                .map(entity -> MagicConverter.convert(entity, FileModel.class, new FileModelMapper()))
+                .collect(Collectors.toList());
+    }
+
+
+    public List<String> getDossierStatusIds(String dossierId, boolean includeDeleted) {
+
+        return fileStatusPersistenceService.findAllByDossierId(dossierId, includeDeleted);
+    }
+
+
+    @Transactional
+    public List<FileModel> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) {
+
+        var fileModels = fileStatusPersistenceService.findAllDossierIdAndIds(dossierId, fileIds)
+                .stream()
+                .map(entity -> MagicConverter.convert(entity, FileModel.class, new FileModelMapper()))
+                .toList();
+
+        return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModels);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FilterByPermissionsService.java

@@ -7,6 +7,7 @@ import org.springframework.security.access.prepost.PostAuthorize;
 import org.springframework.security.access.prepost.PreFilter;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.IHavingDossierId;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 
 /*
@@ -17,6 +18,20 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 @Service
 public class FilterByPermissionsService {
 
+    @PreFilter("hasPermission(filterObject.getDossierId(), 'Dossier', 'VIEW_OBJECT')")
+    public <T extends IHavingDossierId> List<T> onlyViewableHavingDossierId(List<T> items) {
+
+        return items;
+
+    }
+
+    @PreFilter("hasPermission(filterObject, 'Dossier', 'VIEW_OBJECT')")
+    public Set<String> onlyViewableDossierIds(Set<String> dossierIds) {
+
+        return dossierIds;
+
+    }
+
     @PreFilter("hasPermission(filterObject, 'Dossier', 'VIEW_OBJECT')")
     public List<String> onlyViewableDossierIds(List<String> dossierIds) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java

@@ -60,8 +60,9 @@ public class ReanalysisRequiredStatusService {
         Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>();
         Map<String, Long> dossierVersionMap = new HashMap<>();
         Map<String, DossierEntity> dossierMap = new HashMap<>();
+        Map<String, Map<String, Integer>> componentMappingVersionMap = new HashMap<>();
         fileModels.forEach(entry -> {
-            var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
+            var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap, componentMappingVersionMap);
             entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
             entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
             entry.setComponentReanalysisRequired(analysisRequiredResult.isComponentReanalysisRequired());
@@ -75,7 +76,8 @@ public class ReanalysisRequiredStatusService {
                                                            boolean ignoreProcessingStates,
                                                            Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
                                                            Map<String, Long> dossierVersionMap,
-                                                           Map<String, DossierEntity> dossierMap) {
+                                                           Map<String, DossierEntity> dossierMap,
+                                                           Map<String, Map<String, Integer>> componentMappingVersionMap) {
 
         // enhance with dossierTemplateId
         DossierEntity dossier;
@@ -118,7 +120,7 @@ public class ReanalysisRequiredStatusService {
                     if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) {
                         return new AnalysisRequiredResult(true, false);
                     } else {
-                        return requiresReanalysisBasedOnVersionDifference(fileStatus, dossier, dossierTemplateVersionMap, dossierVersionMap);
+                        return requiresReanalysisBasedOnVersionDifference(fileStatus, dossier, dossierTemplateVersionMap, dossierVersionMap,componentMappingVersionMap);
                     }
                 default:
                     return new AnalysisRequiredResult(false, false);
@@ -132,15 +134,17 @@ public class ReanalysisRequiredStatusService {
     private AnalysisRequiredResult requiresReanalysisBasedOnVersionDifference(FileModel fileStatus,
                                                                               DossierEntity dossier,
                                                                               Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
-                                                                              Map<String, Long> dossierVersionMap) {
+                                                                              Map<String, Long> dossierVersionMap,
+                                                                              Map<String, Map<String,Integer>> componentMappingVersionMap) {
 
         // get relevant versions
         Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
                                                                                                    k -> buildVersionData(dossier.getDossierTemplateId()));
 
-        Map<String, Integer> componentMappingVersions = componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId())
-                .stream()
-                .collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion));
+        Map<String, Integer> componentMappingVersions = componentMappingVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
+                                                                                                   k -> componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId())
+                                                                                                           .stream()
+                                                                                                           .collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion)));
 
         Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId()));
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java

@@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionEntrySource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
@@ -108,7 +109,7 @@ public class ManualRedactionMapper {
 
             for (EntityLogEntry entityLogEntry : entityLogEntriesWithId) {
 
-                if (invalidDictionaryRequest(removeRedactionRequest, entityLogEntry) || invalidLocalRequest(removeRedactionRequest, entityLogEntry)) {
+                if (entityLogEntry.getState().equals(EntryState.PENDING) || entityLogEntry.getState().equals(EntryState.REMOVED)) {
                     continue;
                 }
 
@@ -135,28 +136,10 @@ public class ManualRedactionMapper {
     }
 
 
-    private static boolean invalidLocalRequest(RemoveRedactionRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) {
-
-        return !isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.REMOVED);
-    }
-
-
-    private static boolean isDictionaryRequest(RemoveRedactionRequestModel removeRedactionRequest) {
-
-        return removeRedactionRequest.isRemoveFromDictionary() || removeRedactionRequest.isRemoveFromAllDossiers();
-    }
-
-
-    private static boolean invalidDictionaryRequest(RemoveRedactionRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) {
-
-        return isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.PENDING);
-    }
-
-
     public List<RequestEntryPair<ForceRedactionRequest>> toForceRedactionRequestList(String dossierId,
                                                                                      String fileId,
                                                                                      Set<ForceRedactionRequestModel> forceRedactionRequests,
-                                                                                     Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
+                                                                                     Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
 
         List<RequestEntryPair<ForceRedactionRequest>> requests = new ArrayList<>();
 
@@ -172,9 +155,8 @@ public class ManualRedactionMapper {
                     .build();
 
             if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !entityLogEntry.getEngines().contains(Engine.IMPORTED) && entryIsEntityType(entityLogEntry)) {
-                request.setBasedOnDictAnnotationId(forceRedactionRequestModel.getAnnotationId());
                 entityLogEntry.setId(uuid);
-                manualRedactionEntryConsumer.accept(entityLogEntry);
+                manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, forceRedactionRequestModel.getAnnotationId()));
                 request.setAnnotationId(uuid);
             }
 
@@ -188,7 +170,7 @@ public class ManualRedactionMapper {
     public List<RequestEntryPair<LegalBasisChangeRequest>> toLegalBasisChangeRequestList(String dossierId,
                                                                                          String fileId,
                                                                                          Set<LegalBasisChangeRequestModel> legalBasisChangeRequests,
-                                                                                         Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
+                                                                                         Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
 
         List<RequestEntryPair<LegalBasisChangeRequest>> requests = new ArrayList<>();
 
@@ -206,9 +188,8 @@ public class ManualRedactionMapper {
                     .build();
 
             if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !entityLogEntry.getEngines().contains(Engine.IMPORTED) && entryIsEntityType(entityLogEntry)) {
-                request.setBasedOnDictAnnotationId(legalBasisChangeRequest.getAnnotationId());
                 entityLogEntry.setId(uuid);
-                manualRedactionEntryConsumer.accept(entityLogEntry);
+                manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, legalBasisChangeRequest.getAnnotationId()));
                 request.setAnnotationId(uuid);
             }
 
@@ -224,7 +205,7 @@ public class ManualRedactionMapper {
                                                                                          String dossierTemplateId,
                                                                                          Set<RecategorizationRequestModel> recategorizationRequests,
                                                                                          boolean includeUnprocessed,
-                                                                                         Consumer<EntityLogEntry> manualRedactionEntryConsumer) {
+                                                                                         Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer) {
 
         List<EntityLogEntry> entityLogEntries = entityLogMongoWrapperService.getEntityLogEntriesByIds(dossierId,
                                                                                                       fileId,
@@ -247,7 +228,7 @@ public class ManualRedactionMapper {
 
             for (EntityLogEntry entityLogEntry : entityLogEntriesById) {
 
-                if (invalidDictionaryRequest(recategorizationRequest, entityLogEntry) || invalidLocalRequest(recategorizationRequest, entityLogEntry)) {
+                if (entityLogEntry.getState().equals(EntryState.PENDING) || entityLogEntry.getState().equals(EntryState.REMOVED)) {
                     continue;
                 }
 
@@ -291,9 +272,8 @@ public class ManualRedactionMapper {
                     && !recategorizationRequest.isAddToAllDossiers()
                     && !recategorizationRequest.isAddToDictionary()
                     && entryIsEntityType(entityLogEntry)) {
-                    request.setBasedOnDictAnnotationId(recategorizationRequest.getAnnotationId());
                     entityLogEntry.setId(uuid);
-                    manualRedactionEntryConsumer.accept(entityLogEntry);
+                    manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, recategorizationRequest.getAnnotationId()));
                     request.setAnnotationId(uuid);
                 }
 
@@ -305,24 +285,6 @@ public class ManualRedactionMapper {
     }
 
 
-    private static boolean invalidLocalRequest(RecategorizationRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) {
-
-        return !isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.REMOVED);
-    }
-
-
-    private static boolean isDictionaryRequest(RecategorizationRequestModel removeRedactionRequest) {
-
-        return removeRedactionRequest.isAddToDictionary() || removeRedactionRequest.isAddToAllDossiers();
-    }
-
-
-    private static boolean invalidDictionaryRequest(RecategorizationRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) {
-
-        return isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.PENDING);
-    }
-
-
     private void checkSectionLength(String changedSection) {
 
         if (changedSection == null) {
@@ -353,7 +315,7 @@ public class ManualRedactionMapper {
     public List<RequestEntryPair<ResizeRedactionRequest>> toResizeRedactionRequestList(String dossierId,
                                                                                        String fileId,
                                                                                        Set<ResizeRedactionRequestModel> resizeRedactionRequests,
-                                                                                       Consumer<EntityLogEntry> manualRedactionEntryConsumer,
+                                                                                       Consumer<ManualRedactionEntrySource> manualRedactionEntryConsumer,
                                                                                        boolean includeUnprocessed) {
 
         List<EntityLogEntry> entityLogEntries = entityLogMongoWrapperService.getEntityLogEntriesByIds(dossierId,
@@ -366,8 +328,11 @@ public class ManualRedactionMapper {
         List<RequestEntryPair<ResizeRedactionRequest>> requests = new ArrayList<>();
 
         for (ResizeRedactionRequestModel resizeRedactionRequest : resizeRedactionRequests) {
+            for (EntityLogEntry entityLogEntry : entityLogEntries) {
 
-            entityLogEntries.forEach(entityLogEntry -> {
+                if (entityLogEntry.getState().equals(EntryState.PENDING) || entityLogEntry.getState().equals(EntryState.REMOVED)) {
+                    continue;
+                }
                 String uuid = UUID.randomUUID().toString();
 
                 ResizeRedactionRequest request = ResizeRedactionRequest.builder()
@@ -385,14 +350,13 @@ public class ManualRedactionMapper {
                     && !request.isAddToAllDossiers()
                     && !request.getUpdateDictionary()
                     && entryIsEntityType(entityLogEntry)) {
-                    request.setBasedOnDictAnnotationId(resizeRedactionRequest.getAnnotationId());
                     entityLogEntry.setId(uuid);
-                    manualRedactionEntryConsumer.accept(entityLogEntry);
+                    manualRedactionEntryConsumer.accept(new ManualRedactionEntrySource(entityLogEntry, resizeRedactionRequest.getAnnotationId()));
                     request.setAnnotationId(uuid);
                 }
 
                 requests.add(RequestEntryPair.<ResizeRedactionRequest>builder().request(request).entityLogEntry(entityLogEntry).build());
-            });
+            }
 
         }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java

@@ -58,6 +58,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionEntrySource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
@@ -637,8 +638,9 @@ public class ManualRedactionService {
     }
 
 
-    private void addManualRedactionEntry(String fileId, EntityLogEntry entityLogEntry) {
+    private void addManualRedactionEntry(String fileId, ManualRedactionEntrySource manualRedactionEntrySource) {
 
+        EntityLogEntry entityLogEntry = manualRedactionEntrySource.getEntityLogEntry();
         ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
                 .value(entityLogEntry.getValue())
                 .reason(entityLogEntry.getReason())
@@ -656,6 +658,7 @@ public class ManualRedactionService {
                 .dictionaryEntryType(getDictionaryEntryType(entityLogEntry))
                 .fileId(fileId)
                 .requestDate(OffsetDateTime.now())
+                .sourceId(manualRedactionEntrySource.getSourceId())
                 .build();
 
         addManualRedactionEntries(List.of(manualRedactionEntry), false);
@@ -670,7 +673,7 @@ public class ManualRedactionService {
     }
 
 
-    private Consumer<EntityLogEntry> getEntityLogEntryConsumer(String fileId) {
+    private Consumer<ManualRedactionEntrySource> getEntityLogEntryConsumer(String fileId) {
 
         return entry -> addManualRedactionEntry(fileId, entry);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/PendingEntryFactory.java

@@ -94,7 +94,7 @@ public class PendingEntryFactory {
                 .endOffset(-1)
                 .changes(Collections.emptyList())
                 .manualChanges(manualChanges)
-                .engines(new HashSet<>(Set.of(Engine.DICTIONARY)))
+                .engines(new HashSet<>(Set.of(manualRedactionEntry.isAddToDossierDictionary() ? Engine.DOSSIER_DICTIONARY : Engine.DICTIONARY)))
                 .reference(Collections.emptySet())
                 .importedRedactionIntersections(Collections.emptySet())
                 .build();

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierAttributePersistenceService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
 
+import java.util.Collection;
 import java.util.List;
 
 import jakarta.transaction.Transactional;
@@ -58,6 +59,11 @@ public class DossierAttributePersistenceService {
         return dossierAttributeRepository.findByIdDossierId(dossierId);
     }
 
+    public List<DossierAttributeEntity> getDossierAttributes(Collection<String> dossierIds) {
+
+        return dossierAttributeRepository.findByDossierIds(dossierIds);
+    }
+
 
     public DossierAttributeEntity findOne(String dossierId, String dossierAttributeId) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java

@@ -4,6 +4,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.exceptio
 
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -23,6 +24,9 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntryV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileChangeEntryV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierChange;
 
@@ -179,7 +183,7 @@ public class DossierPersistenceService {
     }
 
 
-    public List<DossierEntity> findAllDossiers(List<String> dossierIds) {
+    public List<DossierEntity> findAllDossiers(Collection<String> dossierIds) {
 
         if (!dossierIds.isEmpty()) {
             return dossierRepository.findAllById(dossierIds);
@@ -195,9 +199,9 @@ public class DossierPersistenceService {
     }
 
 
-    public Set<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
 
-        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId);
+        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId, dossierIds);
     }
 
 
@@ -232,6 +236,19 @@ public class DossierPersistenceService {
     }
 
 
+    public DossierChangeResponseV2 hasChangesSinceV2(OffsetDateTime since) {
+
+        var changedDossiers = dossierRepository.findDossierChangeProjectionByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS));
+        var changedFiles = fileRepository.findFileChangeProjectionByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS));
+
+        var response = new DossierChangeResponseV2();
+        response.setDossierChanges(changedDossiers.stream().map(d -> new DossierChangeEntryV2(d.getId(), d.getLastUpdated())).collect(Collectors.toList()));
+        response.setFileChanges(changedFiles.stream().map(f -> new FileChangeEntryV2(f.getId(), f.getDossierId(), f.getLastUpdated())).collect(Collectors.toList()));
+
+        return response;
+    }
+
+
     public Set<DossierChange> hasChangesSince(OffsetDateTime since) {
 
         var dossiersWithChanges = dossierRepository.findDossierChangeByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS));
@@ -282,4 +299,10 @@ public class DossierPersistenceService {
                 .orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
     }
 
+
+    public List<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
+
+        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -213,12 +214,13 @@ public class FileStatusPersistenceService {
     public void updateStatusErrorInfo(String fileId, FileErrorInfo fileErrorInfo) {
 
         if (fileErrorInfo == null) {
-            fileRepository.updateStatusErrorInfo(fileId, null, null, null, null);
+            fileRepository.updateStatusErrorInfo(fileId, null, null, null, null, null);
         } else {
             fileRepository.updateStatusErrorInfo(fileId,
                                                  fileErrorInfo.getCause().substring(0, Math.min(fileErrorInfo.getCause().length(), 255)),
                                                  fileErrorInfo.getQueue(),
                                                  fileErrorInfo.getService(),
+                                                 fileErrorInfo.getErrorCode(),
                                                  fileErrorInfo.getTimestamp());
         }
 
@@ -233,7 +235,7 @@ public class FileStatusPersistenceService {
         }
         if (processingStatus == ProcessingStatus.PROCESSED) {
             // reset the error info
-            fileRepository.updateStatusErrorInfo(fileId, null, null, null, null);
+            fileRepository.updateStatusErrorInfo(fileId, null, null, null, null, null);
             // In case the file is updated to "processed", "lastProcessed" date should be updated to "now"
             fileRepository.updateProcessingStatus(fileId,
                                                   processingStatus,
@@ -636,6 +638,10 @@ public class FileStatusPersistenceService {
         fileRepository.updateFileModificationDate(fileId, fileManipulationDate);
     }
 
+    @Transactional
+    public void resetErrorCounter(String dossierTemplateId){
+        fileRepository.updateErrorCounter(dossierTemplateId, 0);
+    }
 
     @Transactional
     public void updateHasHighlights(String fileId, boolean hasHighlights) {
@@ -703,5 +709,20 @@ public class FileStatusPersistenceService {
     }
 
 
+    public List<FileEntity> findAllByIds(Set<String> fileIds) {
+
+        return fileRepository.findAllById(fileIds);
+    }
+
+    public List<FileEntity> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) {
+
+        return fileRepository.findAllDossierIdAndIds(dossierId, fileIds);
+    }
+
+
+    public List<String> findAllByDossierId(String dossierId, boolean includeDeleted) {
+
+        return fileRepository.findAllByDossierId(dossierId,  includeDeleted);
+    }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java

@@ -42,6 +42,7 @@ public class AddRedactionPersistenceService {
         manualRedactionEntry.setTypeId(addRedactionRequest.getDictionaryTypeId());
         manualRedactionEntry.setDictionaryEntryType(addRedactionRequest.getDictionaryEntryType());
         manualRedactionEntry.setSection(addRedactionRequest.getSection());
+        manualRedactionEntry.setAddToDossierDictionary(manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToAllDossiers());
 
         return manualRedactionRepository.saveAndFlush(manualRedactionEntry);
 
@@ -159,4 +160,11 @@ public class AddRedactionPersistenceService {
         manualRedactionRepository.undeleteByFileId(fileId, deletionTime);
     }
 
+
+    @Transactional
+    public int updateRequestDate(String fileId, String annotationId, OffsetDateTime requestDate) {
+
+        return manualRedactionRepository.updateRequestDateById(new AnnotationEntityId(annotationId, fileId), requestDate);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RecategorizationPersistenceService.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
-import java.util.Set;
 
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -31,7 +30,7 @@ public class RecategorizationPersistenceService {
         AnnotationEntityId id = new AnnotationEntityId(request.getAnnotationId(), fileId);
 
         ManualRecategorizationEntity entity = recategorizationRepository.findById(id)
-                .orElse(ManualRecategorizationEntity.builder().id(id).build());
+                .orElse(buildEntity(id));
 
         entity.setUser(request.getUser());
 
@@ -56,10 +55,19 @@ public class RecategorizationPersistenceService {
             entity.setValue(request.getValue());
         }
         entity.setProcessedDate(null);
+        entity.setSoftDeletedTime(null);
         return entity;
     }
 
 
+    private static ManualRecategorizationEntity buildEntity(AnnotationEntityId id) {
+
+        var e = new ManualRecategorizationEntity();
+        e.setId(id);
+        return e;
+    }
+
+
     @Transactional
     public void hardDelete(String fileId, String annotationId) {
 
@@ -87,24 +95,32 @@ public class RecategorizationPersistenceService {
                 .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
     }
 
+
     public List<ManualRecategorizationEntity> findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) {
 
         if (options.getAnnotationIds().isEmpty()) {
             return recategorizationRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges());
         }
-        return recategorizationRepository.findByFileIdAndOptionsAndAnnotationIds(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges(), options.getAnnotationIds());
+        return recategorizationRepository.findByFileIdAndOptionsAndAnnotationIds(fileId,
+                                                                                 options.isIncludeDeletions(),
+                                                                                 options.isIncludeOnlyUnprocessed(),
+                                                                                 options.isIncludeDictChanges(),
+                                                                                 options.getAnnotationIds());
     }
 
+
     public int softDeleteByFileIds(List<String> fileIds, OffsetDateTime softDeletionTime) {
 
         return recategorizationRepository.softDeleteByFileIds(fileIds, softDeletionTime);
     }
 
+
     public int deleteByFileIds(List<String> fileIds) {
 
         return recategorizationRepository.deleteByFileIds(fileIds);
     }
 
+
     @Transactional
     public void markAsProcessed(String annotationId, String fileId) {
 
@@ -118,6 +134,7 @@ public class RecategorizationPersistenceService {
         recategorizationRepository.saveAndFlush(recategorizationEntity);
     }
 
+
     public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) {
 
         recategorizationRepository.undeleteByFileId(fileId, deletionTime);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/DossierChangeProjection.java

@@ -0,0 +1,9 @@
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection;
+
+import java.time.OffsetDateTime;
+
+public interface DossierChangeProjection {
+
+    String getId();
+    OffsetDateTime getLastUpdated();
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FileChangeProjection.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection;
+
+import java.time.OffsetDateTime;
+
+public interface FileChangeProjection {
+
+    String getId();
+    String getDossierId();
+    OffsetDateTime getLastUpdated();
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierAttributeRepository.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
 
+import java.util.Collection;
 import java.util.List;
 
 import org.springframework.data.jpa.repository.JpaRepository;
@@ -13,6 +14,8 @@ public interface DossierAttributeRepository extends JpaRepository<DossierAttribu
 
     List<DossierAttributeEntity> findByIdDossierId(String dossierId);
 
+    @Query("SELECT e FROM DossierAttributeEntity e WHERE e.id.dossierId IN :dossierIds")
+    List<DossierAttributeEntity> findByDossierIds(@Param("dossierIds") Collection<String> dossierIds);
 
     @Modifying(clearAutomatically = true, flushAutomatically = true)
     @Query("DELETE FROM DossierAttributeEntity e WHERE e.id.dossierId = :dossierId")

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java

@@ -11,10 +11,14 @@ import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.DossierChangeProjection;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.DossierIdAndStatusProjection;
 
 public interface DossierRepository extends JpaRepository<DossierEntity, String> {
 
+    @Query("select d from DossierEntity d where d.lastUpdated > :since")
+    List<DossierChangeProjection> findDossierChangeProjectionByLastUpdatedIsAfter(@Param("since") OffsetDateTime since);
+
     @Query("select d.id from DossierEntity  d where d.lastUpdated > :since")
     List<String> findDossierChangeByLastUpdatedIsAfter(@Param("since") OffsetDateTime since);
 
@@ -86,8 +90,18 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
     List<DossierEntity> findByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
 
 
-    @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId")
-    Set<String> findIdsByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
+    @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.id in :dossierIds")
+    List<String> findIdsByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId, @Param("dossierIds") Set<String> dossierIds);
+
+
+    @Query("select d.id from DossierEntity d "
+           + "where d.dossierTemplateId = :dossierTemplateId "
+           + "and (d.archivedTime is null or (d.archivedTime is not null and :includeArchived = true)) "
+           + "and (d.softDeletedTime is null or (d.softDeletedTime is not null and :includeDeleted = true)) "
+           + "and d.hardDeletedTime is null")
+    List<String> findIdsByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId,
+                                            @Param("includeArchived") boolean includeArchived,
+                                            @Param("includeDeleted") boolean includeDeleted);
 
 
     @Modifying
@@ -115,4 +129,8 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
     @Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL")
     Optional<String> findDossierNameById(@Param("dossierId") String dossierId);
 
+
+    @Query("select d from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null")
+    List<DossierEntity> findActiveByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
 import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
@@ -12,9 +13,12 @@ import org.springframework.data.repository.query.Param;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.DossierChangeProjection;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileChangeProjection;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
 
@@ -125,12 +129,20 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
                                 @Param("processingErrorCounter") int processingErrorCounter);
 
 
+    @Modifying(clearAutomatically = true)
+    @Query("update FileEntity f set  f.processingErrorCounter = :processingErrorCounter "
+           + "where f.dossierId in (select fe.dossierId from FileEntity fe inner join DossierEntity d on d.id = fe.dossierId where d.dossierTemplateId = :dossierTemplateId) and f.processingStatus = 'ERROR'")
+    void updateErrorCounter(@Param("dossierTemplateId") String dossierTemplateId,
+                           @Param("processingErrorCounter") int processingErrorCounter);
+
+
     @Modifying
-    @Query("update FileEntity f set f.errorCause = :cause, f.errorQueue = :queue, f.errorService = :service, f.errorTimestamp = :timestamp  " + "where f.id = :fileId")
+    @Query("update FileEntity f set f.errorCause = :cause, f.errorQueue = :queue, f.errorService = :service, f.errorTimestamp = :timestamp, f.errorCode = :errorCode where f.id = :fileId")
     void updateStatusErrorInfo(@Param("fileId") String fileId,
                                @Param("cause") String cause,
                                @Param("queue") String queue,
                                @Param("service") String service,
+                               @Param("errorCode") ErrorCode errorCode,
                                @Param("timestamp") OffsetDateTime timestamp);
 
 
@@ -270,6 +282,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     int countSoftDeletedFilesPerDossierId(@Param("dossierId") String dossierId);
 
 
+    @Query("select f from FileEntity f where f.lastUpdated > :since")
+    List<FileChangeProjection> findFileChangeProjectionByLastUpdatedIsAfter(@Param("since") OffsetDateTime since);
+
+
     @Query("select distinct f.dossierId from FileEntity f where f.lastUpdated > :since")
     List<String> findDossierChangeByLastUpdatedIsAfter(@Param("since") OffsetDateTime since);
 
@@ -427,16 +443,23 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     @Query("SELECT f FROM FileEntity f WHERE f.protobufMigrationDone = false")
     List<FileEntity> findByProtobufMigrationDoneFalse(Pageable pageable);
 
-    @Modifying
-    @Query("update FileEntity f set f.protobufMigrationDone = true "
-           + "where f.id in (:fileIds)")
-    void setProtobufMigrationDone(@Param("fileIds") List<String> fileIds);
 
+    @Modifying
+    @Query("update FileEntity f set f.protobufMigrationDone = true " + "where f.id in (:fileIds)")
+    void setProtobufMigrationDone(@Param("fileIds") List<String> fileIds);
 
 
     @Query("SELECT f FROM FileEntity f WHERE f.dossierId = :dossierId AND f.hardDeletedTime IS NULL AND f.deleted IS NULL")
     List<DossierStatsFileProjection> findDossierStatsProjectionFileProjectionByDossierId(@Param("dossierId") String dossierId);
 
+
+    @Query("SELECT f.id FROM FileEntity f WHERE f.dossierId = :dossierId AND f.hardDeletedTime IS NULL AND (f.deleted IS NULL or (f.deleted is not null and  :includeDeleted = true))")
+    List<String> findAllByDossierId(@Param("dossierId") String dossierId, @Param("includeDeleted") boolean includeDeleted);
+
+
+    @Query("SELECT f FROM FileEntity f WHERE f.id in :fileIds AND f.dossierId = :dossierId")
+    List<FileEntity> findAllDossierIdAndIds(@Param("dossierId") String dossierId, @Param("fileIds") Set<String> fileIds);
+
 }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java

@@ -51,6 +51,7 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
                                                             @Param("unprocessed") boolean unprocessed,
                                                             @Param("includeDictChanges") boolean includeDictChanges);
 
+
     @Query("""
             select m
             from ManualRedactionEntryEntity m
@@ -61,27 +62,34 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
             and m.id.annotationId in (:annotationIds)
             """)
     List<ManualRedactionEntryEntity> findByFileIdAndOptionsAndAnnotationIds(@Param("fileId") String fileId,
-                                                            @Param("includeDeletions") boolean includeDeletions,
-                                                            @Param("unprocessed") boolean unprocessed,
-                                                            @Param("includeDictChanges") boolean includeDictChanges,
-                                                            @Param("annotationIds") List<String> annotationIds);
+                                                                            @Param("includeDeletions") boolean includeDeletions,
+                                                                            @Param("unprocessed") boolean unprocessed,
+                                                                            @Param("includeDictChanges") boolean includeDictChanges,
+                                                                            @Param("annotationIds") List<String> annotationIds);
+
 
     @Modifying
     @Query("update ManualRedactionEntryEntity m set m.softDeletedTime = :softDeletedTime where m.id.fileId in (:fileIds) and m.softDeletedTime is null")
-    int softDeleteByFileIds(@Param("fileIds") List<String> fileIds,
-                            @Param("softDeletedTime") OffsetDateTime softDeletedTime);
+    int softDeleteByFileIds(@Param("fileIds") List<String> fileIds, @Param("softDeletedTime") OffsetDateTime softDeletedTime);
+
 
     @Modifying
     @Query("update ManualRedactionEntryEntity m set m.softDeletedTime = null where m.id.fileId = :fileId and m.softDeletedTime >= :softDeletedTime")
-    int undeleteByFileId(@Param("fileId") String fileId,
-                            @Param("softDeletedTime") OffsetDateTime softDeletedTime);
+    int undeleteByFileId(@Param("fileId") String fileId, @Param("softDeletedTime") OffsetDateTime softDeletedTime);
+
 
     @Modifying
     @Query("delete ManualRedactionEntryEntity m where m.id.fileId in (:fileIds)")
     int deleteByFileIds(@Param("fileIds") List<String> fileIds);
 
+
     @Modifying
     @Query("update ManualRedactionEntryEntity m set m.processedDate = :processedDate where m.id = :annotationEntityId")
     void markAsProcessed(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("processedDate") OffsetDateTime processedDate);
 
+
+    @Modifying
+    @Query("update ManualRedactionEntryEntity e set e.requestDate = :requestDate where e.id = :annotationEntityId")
+    int updateRequestDateById(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("requestDate") OffsetDateTime requestDate);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java

@@ -18,7 +18,7 @@ public class FileModelMapper implements BiConsumer<FileEntity, FileModel> {
 
         fileEntity.getFileAttributes()
                 .forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue()));
-        fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp()));
+        fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp(), fileEntity.getErrorCode()));
         fileModel.setComponentMappingVersions(getComponentMappingVersions(fileEntity));
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml

@@ -230,8 +230,18 @@ databaseChangeLog:
   - include:
       file: db/changelog/tenant/143-modify-download-redaction-file-status-details.yaml
   - include:
-      file: db/changelog/tenant/144-add-protobuf-migration-done-to-file.yaml
+      file: db/changelog/tenant/144-add-error-code-to-file.yaml
   - include:
-      file: db/changelog/tenant/145-add-indexes-to-file-table.yaml
+      file: db/changelog/tenant/145-add-protobuf-migration-done-to-file.yaml
   - include:
-      file: db/changelog/tenant/146-add-layout-parsing-type-to-dossier-template.yaml
+      file: db/changelog/tenant/146-add-indexes-to-file-table.yaml
+  - include:
+      file: db/changelog/tenant/147-add-layout-parsing-type-to-dossier-template.yaml
+  - include:
+      file: db/changelog/tenant/148-add-quotechar-to-component-mapping.yaml
+  - include:
+      file: db/changelog/tenant/149-remove-based-on-dict-annotation-id-columns.yaml
+  - include:
+      file: db/changelog/tenant/149-add-indexes-across-tables-for-performance.yaml
+  - include:
+      file: db/changelog/tenant/150-add-component-mapping-indexes.yaml

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/144-add-error-code-to-file.yaml

@@ -0,0 +1,11 @@
+databaseChangeLog:
+  - changeSet:
+      id: 144-add-error-code-to-file
+      author: dom
+      changes:
+        - addColumn:
+            columns:
+              - column:
+                  name: error_code
+                  type: VARCHAR(255)
+            tableName: file

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/148-add-quotechar-to-component-mapping.yaml

@@ -0,0 +1,14 @@
+databaseChangeLog:
+  - changeSet:
+      id: add-quote_char-to-component-mapping
+      author: kilian
+      changes:
+        - addColumn:
+            tableName: component_mappings
+            columns:
+              - column:
+                  name: quote_char
+                  type: char
+                  defaultValue: '"'
+                  constraints:
+                    nullable: false

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/149-add-indexes-across-tables-for-performance.yaml

@@ -0,0 +1,220 @@
+databaseChangeLog:
+  - changeSet:
+      id: create_index_if_not_exists_idx_file_last_updated
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_file_last_updated
+              tableName: file
+      changes:
+        - createIndex:
+            tableName: file
+            indexName: idx_file_last_updated
+            columns:
+              - column:
+                  name: last_updated
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_file_deleted_hard_deleted_time
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_file_deleted_hard_deleted_time
+              tableName: file
+      changes:
+        - createIndex:
+            tableName: file
+            indexName: idx_file_deleted_hard_deleted_time
+            columns:
+              - column:
+                  name: deleted
+              - column:
+                  name: hard_deleted_time
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_file_dossier_id_deleted_hard_deleted_time
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_file_dossier_id_deleted_hard_deleted_time
+              tableName: file
+      changes:
+        - createIndex:
+            tableName: file
+            indexName: idx_file_dossier_id_deleted_hard_deleted_time
+            columns:
+              - column:
+                  name: dossier_id
+              - column:
+                  name: deleted
+              - column:
+                  name: hard_deleted_time
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_dossier_last_updated
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_dossier_last_updated
+              tableName: dossier
+      changes:
+        - createIndex:
+            tableName: dossier
+            indexName: idx_dossier_last_updated
+            columns:
+              - column:
+                  name: last_updated
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_dossier_soft_deleted_time_hard_deleted_time
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_dossier_soft_deleted_time_hard_deleted_time
+              tableName: dossier
+      changes:
+        - createIndex:
+            tableName: dossier
+            indexName: idx_dossier_soft_deleted_time_hard_deleted_time
+            columns:
+              - column:
+                  name: soft_deleted_time
+              - column:
+                  name: hard_deleted_time
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_dossier_id_soft_deleted_time_hard_deleted_time
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_dossier_id_soft_deleted_time_hard_deleted_time
+              tableName: dossier
+      changes:
+        - createIndex:
+            tableName: dossier
+            indexName: idx_dossier_id_soft_deleted_time_hard_deleted_time
+            columns:
+              - column:
+                  name: id
+              - column:
+                  name: soft_deleted_time
+              - column:
+                  name: hard_deleted_time
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_dossier_soft_deleted_time_hard_deleted_time_archived_time
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_dossier_soft_deleted_time_hard_deleted_time_archived_time
+              tableName: dossier
+      changes:
+        - createIndex:
+            tableName: dossier
+            indexName: idx_dossier_soft_deleted_time_hard_deleted_time_archived_time
+            columns:
+              - column:
+                  name: soft_deleted_time
+              - column:
+                  name: hard_deleted_time
+              - column:
+                  name: archived_time
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_dossier_dossier_template_id_soft_deleted_time_hard_deleted_time_archived_time
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_dossier_dossier_template_id_soft_deleted_time_hard_deleted_time_archived_time
+              tableName: dossier
+      changes:
+        - createIndex:
+            tableName: dossier
+            indexName: idx_dossier_dossier_template_id_soft_deleted_time_hard_deleted_time_archived_time
+            columns:
+              - column:
+                  name: dossier_template_id
+              - column:
+                  name: soft_deleted_time
+              - column:
+                  name: hard_deleted_time
+              - column:
+                  name: archived_time
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_notification_preference_user_id_in_app_notifications_enabled
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_notification_preference_user_id_in_app_notifications_enabled
+              tableName: notification_preference
+      changes:
+        - createIndex:
+            tableName: notification_preference
+            indexName: idx_notification_preference_user_id_in_app_notifications_enabled
+            columns:
+              - column:
+                  name: user_id
+              - column:
+                  name: in_app_notifications_enabled
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_notification_user_id_creation_date_soft_deleted
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_notification_user_id_creation_date_soft_deleted
+              tableName: notification
+      changes:
+        - createIndex:
+            tableName: notification
+            indexName: idx_notification_user_id_creation_date_soft_deleted
+            columns:
+              - column:
+                  name: user_id
+              - column:
+                  name: creation_date
+              - column:
+                  name: soft_deleted
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_entity_dossier_template_id
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_entity_dossier_template_id
+              tableName: entity
+      changes:
+        - createIndex:
+            tableName: entity
+            indexName: idx_entity_dossier_template_id
+            columns:
+              - column:
+                  name: dossier_template_id
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_entity_dossier_id
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_entity_dossier_id
+              tableName: entity
+      changes:
+        - createIndex:
+            tableName: entity
+            indexName: idx_entity_dossier_id
+            columns:
+              - column:
+                  name: dossier_id

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/149-remove-based-on-dict-annotation-id-columns.yaml

@@ -0,0 +1,21 @@
+databaseChangeLog:
+  - changeSet:
+      id: remove-based-on-dict-annotation-id-column
+      author: corina
+      changes:
+        - dropColumn:
+            columns:
+              - name: based_on_dict_annotation_id
+            tableName: manual_force_redaction
+        - dropColumn:
+            columns:
+              - name: based_on_dict_annotation_id
+            tableName: manual_resize_redaction
+        - dropColumn:
+            columns:
+              - name: based_on_dict_annotation_id
+            tableName: manual_legal_basis_change
+        - dropColumn:
+            columns:
+              - name: based_on_dict_annotation_id
+            tableName: manual_recategorization

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/150-add-component-mapping-indexes.yaml

@@ -0,0 +1,34 @@
+databaseChangeLog:
+  - changeSet:
+      id: create_index_if_not_exists_idx_component_mappings_dossier_template_id
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_component_mappings_dossier_template_id
+              tableName: component_mappings
+      changes:
+        - createIndex:
+            tableName: component_mappings
+            indexName: idx_component_mappings_dossier_template_id
+            columns:
+              - column:
+                  name: dossier_template_id
+
+  - changeSet:
+      id: create_index_if_not_exists_idx_component_mappings_dossier_template_id_name
+      author: Timo
+      preConditions:
+        - not:
+            indexExists:
+              indexName: idx_component_mappings_dossier_template_id_name
+              tableName: component_mappings
+      changes:
+        - createIndex:
+            tableName: component_mappings
+            indexName: idx_component_mappings_dossier_template_id_name
+            columns:
+              - column:
+                  name: dossier_template_id
+              - column:
+                  name: name

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java

@@ -99,23 +99,6 @@ public class Application implements ApplicationContextAware {
     }
 
 
-    @Bean
-    @ConditionalOnProperty(value = "cors.enabled", havingValue = "true")
-    public WebMvcConfigurer corsConfigurer() {
-
-        return new WebMvcConfigurer() {
-
-            @Override
-            public void addCorsMappings(CorsRegistry registry) {
-
-                log.info("Cross Origin Requests are enabled !!!");
-                registry.addMapping("/**").allowedOrigins("*").allowedMethods("GET", "POST", "DELETE", "PUT", "HEAD");
-
-            }
-        };
-    }
-
-
     @Bean
     public DictionaryMergeService dictionaryMergeService() {
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentMappingTest.java

@@ -53,7 +53,12 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
                                                       "application/csv",
                                                       IOUtils.toByteArray(new ClassPathResource("files/componentmapping/file.csv").getInputStream()));
 
-        ComponentMappingMetadataModel componentMappingMetadataModel = dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(), mockMultipartFile, "file", "UTF-8", ",");
+        ComponentMappingMetadataModel componentMappingMetadataModel = dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(),
+                                                                                                                  mockMultipartFile,
+                                                                                                                  "file",
+                                                                                                                  "UTF-8",
+                                                                                                                  ",",
+                                                                                                                  "\"");
 
         assertEquals(componentMappingMetadataModel.getFileName(), mockMultipartFile.getOriginalFilename());
 
@@ -65,11 +70,20 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
         when(componentMappingPersistenceService.getEntityById(anyString(), anyString())).thenReturn(ComponentMappingEntity.builder()
                                                                                                             .id(componentMappingMetadataModel.getId())
                                                                                                             .dossierTemplate(new DossierTemplateEntity())
-                                                                                                            .storageId(buildStorageId(dossierTemplate.getId(), componentMappingMetadataModel.getId(), "file", "update_file.csv"))
+                                                                                                            .storageId(buildStorageId(dossierTemplate.getId(),
+                                                                                                                                      componentMappingMetadataModel.getId(),
+                                                                                                                                      "file",
+                                                                                                                                      "update_file.csv"))
                                                                                                             .fileName("update_file.csv")
                                                                                                             .build());
 
-        componentMappingMetadataModel = dossierTemplateExternalClient.updateMapping(dossierTemplate.getId(), componentMappingMetadataModel.getId(), updateMockMultipartFile, "file", "UTF-8", ",");
+        componentMappingMetadataModel = dossierTemplateExternalClient.updateMapping(dossierTemplate.getId(),
+                                                                                    componentMappingMetadataModel.getId(),
+                                                                                    updateMockMultipartFile,
+                                                                                    "file",
+                                                                                    "UTF-8",
+                                                                                    ",",
+                                                                                    "\"");
 
         assertEquals(componentMappingMetadataModel.getFileName(), updateMockMultipartFile.getOriginalFilename());
     }
@@ -86,7 +100,8 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
                                                       "application/csv",
                                                       IOUtils.toByteArray(new ClassPathResource("files/componentmapping/empty.csv").getInputStream()));
 
-        var result = assertThrows(FeignException.class, () -> dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(), mockMultipartFile, "file", "UTF-8", ","));
+        String id = dossierTemplate.getId();
+        var result = assertThrows(FeignException.class, () -> dossierTemplateExternalClient.uploadMapping(id, mockMultipartFile, "file", "UTF-8", ",", "\""));
         assertTrue(result.getMessage().contains("CSV file can not be empty!"));
     }
 
@@ -95,4 +110,5 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest {
 
         return dossierTemplateId + "/" + id + "_" + name + "_" + fileName;
     }
+
 }

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java

@@ -76,7 +76,7 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest
 
         var dossierTemplate = dossierTemplateClient.getDossierTemplate(dossier.getDossierTemplateId());
 
-        var file = fileTesterAndProvider.testAndProvideFile(dossier, "filename");
+        var file = fileTesterAndProvider.testAndProvideFile(dossier, "filename1");
 
         System.out.println("DOSSIER TEMPLATE ID: " + dossierTemplate.getId());
         System.out.println("DOSSIER ID: " + dossier.getId());
@@ -199,7 +199,7 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest
 
         var dossierTemplate = dossierTemplateClient.getDossierTemplate(dossier.getDossierTemplateId());
 
-        var file = fileTesterAndProvider.testAndProvideFile(dossier, "filename");
+        var file = fileTesterAndProvider.testAndProvideFile(dossier, "filename2");
 
         System.out.println("DOSSIER TEMPLATE ID: " + dossierTemplate.getId());
         System.out.println("DOSSIER ID: " + dossier.getId());
@@ -261,7 +261,7 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest
         assertTrue(overrides.getComponentOverrides().isEmpty());
 
         // case 2: re-upload file that was deleted before overrides should also not be returned anymore
-        fileTesterAndProvider.testAndProvideFile(dossier, "filename");
+        fileTesterAndProvider.testAndProvideFile(dossier, "filename3");
 
         var e = assertThrows(FeignException.class, () -> componentClient.getOverrides(dossierTemplate.getId(), dossier.getId(), file.getId()));
         assertEquals(e.status(), 404);

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsAndGetByIdsTest.java

@@ -0,0 +1,138 @@
+package com.iqser.red.service.peristence.v1.server.integration.tests;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.iqser.red.service.peristence.v1.server.integration.client.CustomPermissionClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
+import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
+import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
+import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
+import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider;
+import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
+import com.iqser.red.service.peristence.v1.server.integration.utils.TokenService;
+import com.iqser.red.service.persistence.management.v1.processor.acl.RedPermission;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntryV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileChangeEntryV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.permission.CustomPermissionMappingModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.permission.CustomPermissionModel;
+import org.junit.jupiter.api.Test;
+
+public class DossierStatsAndGetByIdsTest extends AbstractPersistenceServerServiceTest {
+
+    @Autowired
+    private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider;
+
+    @Autowired
+    private FileTesterAndProvider fileTesterAndProvider;
+
+    @Autowired
+    private DossierTesterAndProvider dossierTesterAndProvider;
+
+    @Autowired
+    private CustomPermissionClient customPermissionClient;
+
+    @Autowired
+    private TokenService tokenService;
+
+    @Autowired
+    private UserProvider userProvider;
+
+    @Autowired
+    private DossierClient dossierClient;
+
+    @Autowired
+    private FileClient fileClient;
+
+
+    @Test
+    public void testDossierChangesGetByIdsAndViewPermissions() {
+
+        var start = OffsetDateTime.now();
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        dossierTemplateTesterAndProvider.provideDefaultColors(dossierTemplate.getId());
+        IntStream.range(0, 5)
+                .forEach(x -> dossierTesterAndProvider.provideTestDossier(dossierTemplate, "test dossier: " + x));
+        var allDossiers = dossierClient.getDossiers(true, true);
+
+        var dossiersByIds = dossierClient.getDossiersByIds(new JSONPrimitive<>(allDossiers.stream().map(Dossier::getId).collect(Collectors.toSet())));
+        assertThat(allDossiers).hasSameElementsAs(dossiersByIds.getValue().values());
+
+        // test changes for owner
+        var changes = dossierClient.changesSinceV2(new JSONPrimitive<>(start));
+        assertThat(changes.getDossierChanges().stream().map(DossierChangeEntryV2::getDossierId)).hasSameElementsAs(allDossiers.stream()
+                .map(Dossier::getId)
+                .collect(Collectors.toSet()));
+
+        // test changes for owner with new timestamp
+        var nextChanges = dossierClient.changesSinceV2(new JSONPrimitive<>(OffsetDateTime.now()));
+        assertThat(nextChanges.getDossierChanges()).isEmpty();
+
+        var before = OffsetDateTime.now();
+        var testFile = fileTesterAndProvider.testAndProvideFile(allDossiers.iterator().next(), "testFile");
+        nextChanges = dossierClient.changesSinceV2(new JSONPrimitive<>(before));
+        assertThat(nextChanges.getDossierChanges()).isEmpty();
+        assertThat(nextChanges.getFileChanges().stream().map(FileChangeEntryV2::getFileId)).containsExactly(testFile.getId());
+
+        var filesToLoad = new HashMap<String, Set<String>>();
+        nextChanges.getFileChanges().forEach(fc -> {
+            filesToLoad.put(fc.getDossierId(), Set.of(fc.getFileId()));
+        });
+        var loadedFiles = fileClient.getFilesByIds(new JSONPrimitive<>(filesToLoad));
+        assertThat(loadedFiles.getValue().values().stream().flatMap(Collection::stream).collect(Collectors.toSet())).containsExactly(testFile);
+
+        // test get byId for other user
+        tokenService.setUser(userProvider.getAltUserId(), "secret");
+        dossiersByIds = dossierClient.getDossiersByIds(new JSONPrimitive<>(allDossiers.stream().map(Dossier::getId).collect(Collectors.toSet())));
+        assertThat(allDossiers).hasSameElementsAs(dossiersByIds.getValue().values());
+
+        // test changes for other user
+        var changesUser2 = dossierClient.changesSinceV2(new JSONPrimitive<>(start));
+        assertThat(changesUser2.getDossierChanges().stream().map(DossierChangeEntryV2::getDossierId)).hasSameElementsAs(allDossiers.stream()
+                .map(Dossier::getId)
+                .collect(Collectors.toSet()));
+
+        // test changes for other user with new timestamp
+        var nextChangesUser2 = dossierClient.changesSinceV2(new JSONPrimitive<>(OffsetDateTime.now()));
+        assertThat(nextChangesUser2.getDossierChanges()).isEmpty();
+
+        setPermissionsNobodyExceptOwnerCanView();
+
+        dossiersByIds = dossierClient.getDossiersByIds(new JSONPrimitive<>(allDossiers.stream().map(Dossier::getId).collect(Collectors.toSet())));
+        assertThat(dossiersByIds.getValue().values()).isEmpty();
+
+        // no visible changes either after permissions removed
+        changesUser2 = dossierClient.changesSinceV2(new JSONPrimitive<>(start));
+        assertThat(changesUser2.getDossierChanges()).isEmpty();
+
+    }
+
+
+    private void setPermissionsNobodyExceptOwnerCanView() {
+
+        CustomPermissionModel targetViewPermission = new CustomPermissionModel(RedPermission.VIEW_OBJECT.getMask(),
+                RedPermission.VIEW_OBJECT.getName(),
+                RedPermission.VIEW_OBJECT.getSort(),
+                false);
+        List<CustomPermissionModel> mappedViewPermissions = new ArrayList<>();
+        List<CustomPermissionMappingModel> customPermissionMappingModels = new ArrayList<>();
+        customPermissionMappingModels.add(new CustomPermissionMappingModel(targetViewPermission, mappedViewPermissions));
+
+        customPermissionClient.saveCustomPermissionMappings("Dossier", customPermissionMappingModels);
+        customPermissionClient.syncAllCustomPermissions();
+    }
+
+}

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateImportTest.java

@@ -149,6 +149,24 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT
         assertThat(systemManagedTypes.size()).isEqualTo(8);
     }
 
+    @SneakyThrows
+    @Test
+    public void testDossierTemplateImportAsUpdateDM() {
+
+        TenantContext.setTenantId("redaction");
+
+        var archive = loadFileFromClasspath("dm.zip");
+        var request = ImportDossierTemplateRequest.builder().archive(archive).updateExistingDossierTemplate(false).userId("system").build();
+        DossierTemplate dossierTemplate = dossierTemplateManagementService.importDossierTemplate(request);
+
+        TypeResponse types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true);
+        List<TypeValue> systemManagedTypes = types.getTypes().stream().filter(TypeValue::isSystemManaged).collect(Collectors.toList());
+        assertThat(systemManagedTypes.size()).isEqualTo(8);
+
+        var request1 = ImportDossierTemplateRequest.builder().archive(archive).dossierTemplateId(dossierTemplate.getId()).updateExistingDossierTemplate(true).userId("system").build();
+        dossierTemplateManagementService.importDossierTemplate(request1);
+    }
+
     @SneakyThrows
     @Test
     public void testDossierTemplateImportWithTheSameName() {

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java

@@ -173,7 +173,7 @@ public class EntityLogMergeTest {
                              .get(0).getManualRedactionType(), ManualRedactionType.REMOVE);
         assertEquals(removeEntryLogEntry.getChanges()
                              .get(0).getType(), ChangeType.IGNORED);
-        assertTrue(removeEntryLogEntry.getEngines().contains(Engine.MANUAL));
+        assertFalse(removeEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
         var optionalResizeEntryLogEntry = response.getEntityLogEntry()
                 .stream()
@@ -203,7 +203,7 @@ public class EntityLogMergeTest {
         assertEquals(resizeEntryLogEntry.getChanges()
                              .get(0).getPropertyChanges()
                              .get("positions"), "[1.0, 1.0, 1.0, 1.0, 1] -> [2.0, 2.0, 2.0, 2.0, 1]");
-        assertTrue(resizeEntryLogEntry.getEngines().contains(Engine.MANUAL));
+//        assertTrue(resizeEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
         var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry()
                 .stream()
@@ -221,7 +221,7 @@ public class EntityLogMergeTest {
         assertEquals(legalBasisEntryLogEntry.getChanges()
                              .get(0).getPropertyChanges()
                              .get("legalBasis"), "legalBasis -> New legal basis");
-        assertTrue(legalBasisEntryLogEntry.getEngines().contains(Engine.MANUAL));
+//        assertTrue(legalBasisEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
         var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry()
                 .stream()
@@ -242,7 +242,7 @@ public class EntityLogMergeTest {
         assertEquals(forceRedactionEntryLogEntry.getChanges()
                              .get(0).getPropertyChanges()
                              .get("state"), "SKIPPED -> APPLIED");
-        assertTrue(forceRedactionEntryLogEntry.getEngines().contains(Engine.MANUAL));
+//        assertTrue(forceRedactionEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
         var optionalRectangleEntryLogEntry = response.getEntityLogEntry()
                 .stream()
@@ -278,7 +278,7 @@ public class EntityLogMergeTest {
         assertEquals(recategorizationEntryLogEntry.getChanges()
                              .get(0).getPropertyChanges()
                              .get("type"), "CBI_author -> PII");
-        assertTrue(recategorizationEntryLogEntry.getEngines().contains(Engine.MANUAL));
+//        assertTrue(recategorizationEntryLogEntry.getEngines().contains(Engine.MANUAL));
     }
 
 
@@ -419,8 +419,8 @@ public class EntityLogMergeTest {
         String localId = UUID.randomUUID().toString();
 
         ManualRedactions manualRedactions = ManualRedactions.builder()
-                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
-                .resizeRedactions(Set.of(provideManualResize(localId, "Darth", dictEntryToResizeId)))
+                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", dictEntryToResizeId)))
+                .resizeRedactions(Set.of(provideManualResize(localId, "Darth")))
                 .build();
 
         var entityLog = provideEntityLog(entryToRemoveId, dictEntryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, true);
@@ -457,6 +457,59 @@ public class EntityLogMergeTest {
     }
 
 
+    @Test
+    public void testMergeEntityLogWithManualRecatBasedOnDictRedaction() {
+
+        String dossierId = "dossierId";
+        String dossierTemplateId = "dossierTemplateId";
+
+        String entryToRemoveId = UUID.randomUUID().toString();
+        String dictEntryToResizeId = UUID.randomUUID().toString();
+        String entryLegalBasisId = UUID.randomUUID().toString();
+        String forceRedactionId = UUID.randomUUID().toString();
+        String entryToRecategorizeId = UUID.randomUUID().toString();
+
+        String localId = UUID.randomUUID().toString();
+
+        ManualRedactions manualRedactions = ManualRedactions.builder()
+                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", entryToRecategorizeId)))
+                .recategorizations(Set.of(provideManualRecat(localId, "Darth Vader")))
+                .build();
+
+        var entityLog = provideEntityLog(entryToRemoveId, dictEntryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, true);
+
+        when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
+        when(fileStatusService.getStatus(FILE_ID)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(FILE_ID).build());
+        when(fileManagementStorageService.getEntityLog(dossierId, FILE_ID)).thenReturn(entityLog);
+        when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+        when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
+        when(fileStatusPersistenceService.getStatus(FILE_ID)).thenReturn(FileEntity.builder().id(FILE_ID).fileAttributes(Collections.emptyList()).build());
+        when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
+
+        EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+
+        assertNotNull(response);
+        assertFalse(response.getEntityLogEntry().isEmpty());
+
+        var optionalRecatEntryLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRecategorizeId))
+                .findFirst();
+        assertTrue(optionalRecatEntryLogEntry.isPresent());
+        assertEquals(EntryType.ENTITY, optionalRecatEntryLogEntry.get().getEntryType());
+        assertEquals(EntryState.REMOVED, optionalRecatEntryLogEntry.get().getState());
+
+        var optionalRecatEntryLogEntry2 = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getId().equals(localId))
+                .findFirst();
+        assertTrue(optionalRecatEntryLogEntry2.isPresent());
+        assertEquals(EntryType.ENTITY, optionalRecatEntryLogEntry2.get().getEntryType());
+        assertEquals(EntryState.APPLIED, optionalRecatEntryLogEntry2.get().getState());
+        assertFalse(optionalRecatEntryLogEntry.get().getId().equals(optionalRecatEntryLogEntry2.get().getId()));
+    }
+
+
     //dict entry, resize local, remove local
     @Test
     public void testMergeEntityLogWithManualResizeAndRemoveChangesOnDictRedaction() {
@@ -473,8 +526,8 @@ public class EntityLogMergeTest {
         String localId = UUID.randomUUID().toString();
 
         ManualRedactions manualRedactions = ManualRedactions.builder()
-                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
-                .resizeRedactions(Set.of(provideManualResize(localId, "Darth", dictEntryToResizeId)))
+                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", dictEntryToResizeId)))
+                .resizeRedactions(Set.of(provideManualResize(localId, "Darth")))
                 .idsToRemove(Set.of(IdRemoval.builder().annotationId(localId).requestDate(OffsetDateTime.now()).user("user").fileId(FILE_ID).build()))
                 .build();
 
@@ -528,7 +581,7 @@ public class EntityLogMergeTest {
         String localId = UUID.randomUUID().toString();
 
         ManualRedactions manualRedactions = ManualRedactions.builder()
-                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
+                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", null)))
                 .idsToRemove(Set.of(IdRemoval.builder().annotationId(localId).requestDate(OffsetDateTime.now()).user("user").fileId(FILE_ID).build()))
                 .build();
 
@@ -573,8 +626,8 @@ public class EntityLogMergeTest {
         String localId = UUID.randomUUID().toString();
 
         ManualRedactions manualRedactions = ManualRedactions.builder()
-                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader")))
-                .resizeRedactions(Set.of(provideManualResize(localId, "Darth", dictEntryToResizeId)))
+                .entriesToAdd(Set.of(provideManualAdd(localId, "Darth Vader", dictEntryToResizeId)))
+                .resizeRedactions(Set.of(provideManualResize(localId, "Darth")))
                 .idsToRemove(Set.of(IdRemoval.builder().annotationId(localId).requestDate(OffsetDateTime.now()).user("user").fileId(FILE_ID).build()))
                 .build();
 
@@ -681,12 +734,7 @@ public class EntityLogMergeTest {
         String entryId = UUID.randomUUID().toString();
 
         ManualRedactions manualRedactions = ManualRedactions.builder()
-                .idsToRemove(Set.of(IdRemoval.builder()
-                                                  .annotationId(entryId)
-                                                  .fileId("file")
-                                                  .requestDate(OffsetDateTime.now())
-                                                  .user("user")
-                                                  .build()))
+                .idsToRemove(Set.of(IdRemoval.builder().annotationId(entryId).fileId("file").requestDate(OffsetDateTime.now()).user("user").build()))
                 .build();
 
         var entityLog = new EntityLog(1,
@@ -725,7 +773,8 @@ public class EntityLogMergeTest {
                 .findFirst();
         assertTrue(optionalEntityLogEntry.isPresent());
         assertEquals(optionalEntityLogEntry.get().getChanges().size(), 1);
-        assertEquals(optionalEntityLogEntry.get().getChanges().get(0).getType(), ChangeType.IGNORED);
+        assertEquals(optionalEntityLogEntry.get().getChanges()
+                             .get(0).getType(), ChangeType.IGNORED);
     }
 
 
@@ -877,7 +926,7 @@ public class EntityLogMergeTest {
     }
 
 
-    private ManualRedactionEntry provideManualAdd(String entryToAddId, String valueToAdd) {
+    private ManualRedactionEntry provideManualAdd(String entryToAddId, String valueToAdd, String sourceId) {
 
         return ManualRedactionEntry.builder()
                 .positions(List.of(new Rectangle(1f, 2f, 3f, 4f, 1)))
@@ -892,11 +941,12 @@ public class EntityLogMergeTest {
                 .dictionaryEntryType(DictionaryEntryType.ENTRY)
                 .type("manual")
                 .user("User")
+                .sourceId(sourceId)
                 .build();
     }
 
 
-    private ManualResizeRedaction provideManualResize(String entryToResizeId, String resizedValue, String dictId) {
+    private ManualResizeRedaction provideManualResize(String entryToResizeId, String resizedValue) {
 
         List<Rectangle> positions = new ArrayList<>();
         positions.add(new Rectangle(2, 2, 2, 2, 1));
@@ -910,9 +960,180 @@ public class EntityLogMergeTest {
                 .addToAllDossiers(false)
                 .user("User")
                 .fileId("file")
-                .basedOnDictAnnotationId(dictId)
                 .build();
     }
 
+    private ManualRecategorization provideManualRecat(String entryToRecatId, String value) {
+
+        return ManualRecategorization.builder()
+                .fileId(FILE_ID)
+                .annotationId(entryToRecatId)
+                .value(value)
+                .requestDate(OffsetDateTime.now())
+                .section("")
+                .addToDictionary(false)
+                .addToAllDossiers(false)
+                .user("User")
+                .fileId("file")
+                .build();
+    }
+
+
+    @Test
+    public void testRecategorizeWithoutCreatingOverlap() {
+
+        String fileId = "fileId";
+        String dossierId = "dossierId";
+        String dossierTemplateId = "dossierTemplateId";
+
+        String entryId = UUID.randomUUID().toString();
+        String dictId = UUID.randomUUID().toString();
+
+        ManualRedactionEntry add = provideManualAdd(entryId, "Sensitive information", dictId);
+        add.setType("CBI_author");
+        ManualRedactions manualRedactions = ManualRedactions.builder()
+                .entriesToAdd(Set.of(add))
+                .recategorizations(Set.of(ManualRecategorization.builder()
+                                                  .annotationId(entryId)
+                                                  .fileId(fileId)
+                                                  .legalBasis("New legal basis")
+                                                  .type("PII")
+                                                  .requestDate(OffsetDateTime.now())
+                                                  .user("User")
+                                                  .build()))
+                .build();
+
+        List<Position> positions = new ArrayList<>();
+        positions.add(new Position(1, 1, 1, 1, 1));
+
+        EntityLogEntry existingEntry = EntityLogEntry.builder()
+                .id(dictId)
+                .type("CBI_author")
+                .value("Sensitive Information")
+                .entryType(EntryType.ENTITY)
+                .state(EntryState.APPLIED)
+                .dictionaryEntry(true)
+                .positions(positions)
+                .legalBasis("Initial legal basis")
+                .build();
+
+        EntityLog entityLog = new EntityLog(1, 1, Lists.newArrayList(existingEntry), Collections.emptyList(), 0, 0, 0, 0);
+
+        when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
+        when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
+        when(fileManagementStorageService.getEntityLog(dossierId, fileId)).thenReturn(entityLog);
+        when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+        when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
+        when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
+        when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
+
+        EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+
+        assertNotNull(response, "The merged EntityLog should not be null.");
+        assertFalse(response.getEntityLogEntry().isEmpty(), "The merged EntityLog should contain entries.");
+
+        var updatedEntryOpt = response.getEntityLogEntry()
+                .stream()
+                .filter(e -> e.getId().equals(entryId))
+                .findFirst();
+
+        assertTrue(updatedEntryOpt.isPresent(), "The recategorized entry should be present in the EntityLog.");
+
+        EntityLogEntry updatedEntry = updatedEntryOpt.get();
+
+        assertEquals("PII", updatedEntry.getType(), "The entry type should be updated to PII.");
+
+        assertEquals("New legal basis", updatedEntry.getLegalBasis(), "The legal basis should be updated.");
+
+        assertEquals(EntryState.APPLIED, updatedEntry.getState(), "The entry state should remain APPLIED.");
+
+        assertEquals(1,
+                     response.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> !entityLogEntry.getState().equals(EntryState.REMOVED))
+                             .count(),
+                     "There should be only one entry in the EntityLog after recategorization.");
+
+    }
+
+
+    @Test
+    public void testForceRedactSkippedCBIAddressAndLocalRemoveWithoutCreatingDuplicate() {
+
+        String fileId = "fileId";
+        String dossierId = "dossierId";
+        String dossierTemplateId = "dossierTemplateId";
+
+        String entryId = UUID.randomUUID().toString();
+        String dictId = UUID.randomUUID().toString();
+
+        ManualRedactionEntry add = provideManualAdd(entryId, "123 Main St", dictId);
+        add.setType("CBI_address");
+        ManualRedactions manualRedactions = ManualRedactions.builder()
+                .entriesToAdd(Set.of(add))
+                .forceRedactions(Set.of(ManualForceRedaction.builder()
+                                                .annotationId(entryId)
+                                                .fileId(fileId)
+                                                .legalBasis("Force")
+                                                .user("User")
+                                                .requestDate(OffsetDateTime.now())
+                                                .build()))
+                .idsToRemove(Set.of(IdRemoval.builder().annotationId(entryId).fileId(fileId).user("User").requestDate(OffsetDateTime.now()).build()))
+                .build();
+
+        List<Position> positions = new ArrayList<>();
+        positions.add(new Position(1, 1, 1, 1, 1));
+
+        EntityLogEntry existingEntry = EntityLogEntry.builder()
+                .id(dictId)
+                .type("CBI_address")
+                .value("123 Main St")
+                .entryType(EntryType.ENTITY)
+                .state(EntryState.SKIPPED)
+                .dictionaryEntry(true)
+                .positions(positions)
+                .legalBasis("Initial legal basis")
+                .build();
+
+        EntityLog entityLog = new EntityLog(1, 1, Lists.newArrayList(existingEntry), Collections.emptyList(), 0, 0, 0, 0);
+        when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
+        when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
+        when(fileManagementStorageService.getEntityLog(dossierId, fileId)).thenReturn(entityLog);
+        when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+        when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
+        when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
+        when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
+
+        EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+
+        assertNotNull(response, "The merged EntityLog should not be null.");
+        assertFalse(response.getEntityLogEntry().isEmpty(), "The merged EntityLog should contain entries.");
+
+        List<EntityLogEntry> relatedEntries = response.getEntityLogEntry()
+                .stream()
+                .filter(e -> e.getId().equals(entryId))
+                .toList();
+
+        assertEquals(1, relatedEntries.size(), "There should be only one entry for the entryId after force redaction and local removal.");
+
+        EntityLogEntry updatedEntry = relatedEntries.get(0);
+
+        assertEquals("CBI_address", updatedEntry.getType(), "The entry type should remain CBI_address after force redaction and local removal.");
+
+        long skippedAnnotations = response.getEntityLogEntry()
+                .stream()
+                .filter(e -> e.getType().equals("manual") && e.getEntryType() == EntryType.AREA && e.getState() == EntryState.SKIPPED)
+                .count();
+
+        assertEquals(0, skippedAnnotations, "No additional skipped annotations (Cross Marks) should be present.");
+
+        var nonRemovedEntries = response.getEntityLogEntry()
+                .stream()
+                .filter(e -> !e.getId().equals(entryId) && !e.getState().equals(EntryState.REMOVED))
+                .toList();
+
+        assertEquals(1, nonRemovedEntries.size(), "There should be only one entry in the EntityLog after force and local remove.");
+    }
+
 }
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java

@@ -28,7 +28,6 @@ import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.core.io.InputStreamResource;
 import org.springframework.mock.web.MockMultipartFile;
 
 import com.github.dockerjava.zerodep.shaded.org.apache.commons.codec.binary.Base64;
@@ -465,28 +464,26 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
 
         manualRedactionClient.removeRedactionBulk(dossierId,
                                                   fileId,
-                                                  Set.of(RemoveRedactionRequestModel.builder().annotationId(annotationId).comment("comment").removeFromDictionary(false).build()),
-                                                  false);
+                                                  Set.of(RemoveRedactionRequestModel.builder().annotationId(annotationId).comment("removeComment").removeFromDictionary(false).build()));
         manualRedactionClient.forceRedactionBulk(dossierId,
                                                  fileId,
-                                                 Set.of(ForceRedactionRequestModel.builder().annotationId("forceRedactionAnnotation").comment("comment").legalBasis("1").build()));
+                                                 Set.of(ForceRedactionRequestModel.builder().annotationId("forceRedactionAnnotation").comment("forceComment").legalBasis("1").build()));
         manualRedactionClient.legalBasisChangeBulk(dossierId,
                                                    fileId,
                                                    Set.of(LegalBasisChangeRequestModel.builder()
                                                                   .annotationId("legalBasisChangeAnnotation")
-                                                                  .comment("comment")
+                                                                  .comment("legalBasisComment")
                                                                   .legalBasis("1")
                                                                   .build()));
         manualRedactionClient.recategorizeBulk(dossierId,
                                                fileId,
                                                Set.of(RecategorizationRequestModel.builder()
                                                               .annotationId(annotationId)
-                                                              .comment("comment")
+                                                              .comment("recategorizationComment")
                                                               .type("new-type")
                                                               .legalBasis(null)
                                                               .section("section")
-                                                              .build()),
-                                               false);
+                                                              .build()));
 
         manualRedactionClient.addComment(dossierId, fileId, commentId, AddCommentRequestModel.builder().text("comment added via text fileId").build());
 
@@ -812,8 +809,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
                                                                                   .legalBasis("")
                                                                                   .section("section")
                                                                                   .value("value entry 3")
-                                                                                  .build()),
-                                                                   false);
+                                                                                  .build()));
 
         var loadedFile = fileClient.getFileStatus(dossierId, fileId);
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java

@@ -19,11 +19,12 @@ import static org.mockito.Mockito.when;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Stream;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -31,7 +32,6 @@ import org.junit.jupiter.api.Test;
 import org.mockito.stubbing.Answer;
 import org.springframework.beans.factory.annotation.Autowired;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient;
@@ -44,6 +44,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvid
 import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider;
 import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRecategorizationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
@@ -51,12 +52,14 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Dossier
 import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService;
 import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
-import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RecategorizationPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.queue.SearchTermOccurrencesResponseReceiver;
 import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionRequest;
+import com.iqser.red.service.persistence.management.v1.processor.utils.DossierMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
@@ -91,6 +94,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Remo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ResizeRedactionRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 
 import feign.FeignException;
 
@@ -121,7 +125,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
     private FileStatusPersistenceService fileStatusPersistenceService;
 
     @Autowired
-    private ObjectMapper objectMapper;
+    private FileStatusService fileStatusService;
 
     @Autowired
     private FileClient fileClient;
@@ -138,15 +142,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
     @Autowired
     private DictionaryManagementService dictionaryManagementService;
 
-    @Autowired
-    private EntryPersistenceService entryPersistenceService;
-
     @Autowired
     private FileProcessingClient fileProcessingClient;
 
-    @Autowired
-    private ManualRedactionService manualRedactionService;
-
     @Autowired
     private ManualRedactionProviderService manualRedactionProviderService;
 
@@ -159,6 +157,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
     @Autowired
     private SearchTermOccurrencesResponseReceiver searchTermOccurrencesResponseReceiver;
 
+    @Autowired
+    private RecategorizationPersistenceService recategorizationPersistenceService;
+
+    @Autowired
+    private AddRedactionPersistenceService addRedactionPersistenceService;
+
 
     @Test
     public void testRemoveToDossierTemplateWithDossierDictionaryOnlyTrue() {
@@ -198,8 +202,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                     .annotationId("AnnotationId")
                                                                                     .removeFromDictionary(true)
                                                                                     .removeFromAllDossiers(true)
-                                                                                    .build()),
-                                                                     false));//.get(0);
+                                                                                    .build())));//.get(0);
 
         var dossierTemplateDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId()), null);
         assertThat(dossierTemplateDictionary.getEntries().size()).isZero();
@@ -391,8 +394,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                  .annotationId("AnnotationId")
                                                                  .removeFromDictionary(true)
                                                                  .removeFromAllDossiers(true)
-                                                                 .build()),
-                                                  false);
+                                                                 .build()));
 
         dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
         assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder("Darth Vader");
@@ -447,8 +449,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
         manualRedactionClient.removeRedactionBulk(dossier.getId(),
                                                   file.getId(),
-                                                  Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()),
-                                                  false).getManualAnnotationResponses()
+                                                  Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()))
+                .getManualAnnotationResponses()
                 .get(0);
 
         var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
@@ -506,8 +508,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                  .annotationId("AnnotationId")
                                                                  .removeFromDictionary(true)
                                                                  .removeFromAllDossiers(true)
-                                                                 .build()),
-                                                  false).getManualAnnotationResponses()
+                                                                 .build())).getManualAnnotationResponses()
                 .get(0);
 
         var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
@@ -627,7 +628,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .addToAllDossiers(true)
                 .build();
 
-        manualRedactionClient.resizeRedactionBulk(dossier1.getId(), file1.getFileId(), Set.of(resizeRedactionDosAndAddToAllDos), false);
+        manualRedactionClient.resizeRedactionBulk(dossier1.getId(), file1.getFileId(), Set.of(resizeRedactionDosAndAddToAllDos));
 
         loadedRedactionsFile1 = manualRedactionClient.getManualRedactions(file1.getDossierId(), file1.getFileId(), false, true);
 
@@ -796,7 +797,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .addToAllDossiers(true)
                 .build();
 
-        manualRedactionClient.resizeRedactionBulk(dossier1.getId(), file1.getFileId(), Set.of(resizeRedactionDosAndAddToAllDos), false);
+        manualRedactionClient.resizeRedactionBulk(dossier1.getId(), file1.getFileId(), Set.of(resizeRedactionDosAndAddToAllDos));
 
         loadedRedactionsFile1 = manualRedactionClient.getManualRedactions(file1.getDossierId(), file1.getFileId(), false, true);
 
@@ -969,7 +970,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .addToAllDossiers(true)
                 .build();
 
-        var resizeRedactions = manualRedactionClient.resizeRedactionBulk(dossier2.getId(), file2.getFileId(), Set.of(resizeRedactionDosTemp), false);
+        var resizeRedactions = manualRedactionClient.resizeRedactionBulk(dossier2.getId(), file2.getFileId(), Set.of(resizeRedactionDosTemp));
 
         var loadedRedactionsFile2 = manualRedactionClient.getManualRedactions(file2.getDossierId(), file2.getFileId(), false, true);
 
@@ -1139,7 +1140,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .addToAllDossiers(true)
                 .build();
 
-        var resizeRedactions = manualRedactionClient.resizeRedactionBulk(dossier2.getId(), file2.getFileId(), Set.of(resizeRedactionDosTemp), false);
+        var resizeRedactions = manualRedactionClient.resizeRedactionBulk(dossier2.getId(), file2.getFileId(), Set.of(resizeRedactionDosTemp));
 
         var loadedRedactionsFile2 = manualRedactionClient.getManualRedactions(file2.getDossierId(), file2.getFileId(), false, true);
 
@@ -1264,8 +1265,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                               .legalBasis("")
                                                               .section("section")
                                                               .value(lukeSkywalker)
-                                                              .build()),
-                                               false);
+                                                              .build()));
 
         dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
         assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder(darthVader);
@@ -1351,8 +1351,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                               .legalBasis("")
                                                               .section("section")
                                                               .value(lukeSkywalker)
-                                                              .build()),
-                                               false);
+                                                              .build()));
 
         dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
         assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder(darthVader);
@@ -1387,7 +1386,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .positions(List.of(Rectangle.builder().page(1).height(1).width(2).topLeftX(1).topLeftY(1).build()))
                 .build();
 
-        ManualRedactionResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(), file.getId(), Set.of(resizeRedactionRequestModel), false);
+        ManualRedactionResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(), file.getId(), Set.of(resizeRedactionRequestModel));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true);
 
@@ -1475,8 +1474,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                               .legalBasis("")
                                                               .section("section")
                                                               .value(lukeSkywalker)
-                                                              .build()),
-                                               false);
+                                                              .build()));
 
         dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
         assertThat(dossierTemplateDictionary.getEntries()).isEmpty();
@@ -1652,8 +1650,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                  .annotationId("AnnotationId")
                                                                  .removeFromDictionary(true)
                                                                  .removeFromAllDossiers(true)
-                                                                 .build()),
-                                                  false);
+                                                                 .build()));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(allManualRedactions.getIdsToRemove().size(), 1);
@@ -1683,8 +1680,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                  .annotationId("AnnotationId2")
                                                                  .removeFromDictionary(true)
                                                                  .removeFromAllDossiers(true)
-                                                                 .build()),
-                                                  false);
+                                                                 .build()));
 
         allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(allManualRedactions.getIdsToRemove().size(), 2);
@@ -1925,8 +1921,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                  .legalBasis("")
                                                                                                  .section("section")
                                                                                                  .type(type.getType())
-                                                                                                 .build()),
-                                                                                  false);
+                                                                                                 .build()));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(allManualRedactions.getRecategorizations().size(), 1);
@@ -1961,8 +1956,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                   .legalBasis("")
                                                                                                   .section("section")
                                                                                                   .type(type.getType())
-                                                                                                  .build()),
-                                                                                   false);
+                                                                                                  .build()));
 
         allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(allManualRedactions.getRecategorizations().size(), 2);
@@ -2274,7 +2268,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .addToAllDossiers(true)
                 .build();
 
-        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel, recatModelNoLegalBasis), false);
+        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel, recatModelNoLegalBasis));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(2, allManualRedactions.getRecategorizations().size());
@@ -2282,7 +2276,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                            .stream()
                            .anyMatch(entry -> entry.getAnnotationId().equals("annotationId")));
 
-        assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAnnotationResponses()
+        assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis)).getManualAnnotationResponses()
                 .get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long");
 
         assertNull(allManualRedactions.getRecategorizations()
@@ -2348,7 +2342,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .section("overriddenSection")
                 .build();
 
-        ManualRedactionResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel), false);
+        ManualRedactionResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(1, allManualRedactions.getRecategorizations().size());
@@ -2399,7 +2393,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .legalBasis("")
                 .build();
 
-        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recategorizationRequestModel), false);
+        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recategorizationRequestModel));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
 
@@ -2495,7 +2489,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .positions(List.of(Rectangle.builder().page(1).height(1).width(2).topLeftX(1).topLeftY(1).build()))
                 .build();
 
-        manualRedactionClient.resizeRedactionBulk(dossier.getId(), file.getId(), Set.of(resizeRedactionRequestModel), false);
+        manualRedactionClient.resizeRedactionBulk(dossier.getId(), file.getId(), Set.of(resizeRedactionRequestModel));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
 
@@ -2652,7 +2646,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .value("Image:Other")
                 .build();
 
-        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recategorizationRequestModel), false);
+        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recategorizationRequestModel));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
 
@@ -2724,7 +2718,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .value("Ranya Eikenboom")
                 .build();
 
-        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recategorizationRequestModel), false);
+        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recategorizationRequestModel));
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
 
@@ -2926,8 +2920,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                      .addToAllDossiers(false)
                                                                                                      .value("Luke")
                                                                                                      .positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1)))
-                                                                                                     .build()),
-                                                                                      false).getManualAnnotationResponses()
+                                                                                                     .build())).getManualAnnotationResponses()
                 .get(0);
 
         assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
@@ -2985,8 +2978,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                   .legalBasis("")
                                                                                                   .section("section")
                                                                                                   .value("Luke Skywalker")
-                                                                                                  .build()),
-                                                                                   false).getManualAnnotationResponses()
+                                                                                                  .build())).getManualAnnotationResponses()
                 .get(0);
 
         assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
@@ -3006,16 +2998,31 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         var file = fileTesterAndProvider.testAndProvideFile(dossier);
 
         var type = typeProvider.testAndProvideType(dossierTemplate);
-        dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("Luke Skywalker"), false, dossier.getId(), DictionaryEntryType.ENTRY);
+        String dictValue = "Crandu Seku Laku Meku";
+        dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of(dictValue), false, dossier.getId(), DictionaryEntryType.ENTRY);
         var entityLog = new EntityLog(1,
                                       1,
                                       List.of(EntityLogEntry.builder()
-                                                      .id("AnnotationId")
-                                                      .type("test")
-                                                      .value("Luke Skywalker")
+                                                      .id("AnnotationId1")
+                                                      .type("CBI_author")
+                                                      .value(dictValue)
                                                       .entryType(EntryType.ENTITY)
                                                       .state(EntryState.APPLIED)
                                                       .dictionaryEntry(true)
+                                                      .dossierDictionaryEntry(true)
+                                                      .engines(Set.of(Engine.DOSSIER_DICTIONARY))
+                                                      .positions(List.of(new Position(new float[]{57f, 446f, 121f, 13f}, 1)))
+                                                      .build(),
+                                              EntityLogEntry.builder()
+                                                      .id("AnnotationId2")
+                                                      .type("CBI_author")
+                                                      .value(dictValue)
+                                                      .entryType(EntryType.ENTITY)
+                                                      .state(EntryState.APPLIED)
+                                                      .dictionaryEntry(true)
+                                                      .dossierDictionaryEntry(true)
+                                                      .engines(Set.of(Engine.DOSSIER_DICTIONARY))
+                                                      .positions(List.of(new Position(new float[]{57f, 473f, 121f, 13f}, 1)))
                                                       .build()),
                                       null,
                                       0,
@@ -3028,19 +3035,19 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         ManualAnnotationResponse response = manualRedactionClient.removeRedactionBulk(dossier.getId(),
                                                                                       file.getId(),
                                                                                       Set.of(RemoveRedactionRequestModel.builder()
-                                                                                                     .annotationId("AnnotationId")
+                                                                                                     .annotationId("AnnotationId1")
                                                                                                      .removeFromDictionary(false)
                                                                                                      .removeFromAllDossiers(false)
-                                                                                                     .build()),
-                                                                                      false).getManualAnnotationResponses()
+                                                                                                     .build())).getManualAnnotationResponses()
                 .get(0);
 
-        assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
+        assertEquals(response.getEntityLogEntry().getId(), "AnnotationId1");
         assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED);
         assertEquals(response.getEntityLogEntry().getManualChanges()
                              .get(0).getManualRedactionType(), ManualRedactionType.REMOVE);
         assertNull(response.getEntityLogEntry().getManualChanges()
                            .get(0).getProcessedDate());
+        assertFalse(response.getEntityLogEntry().getEngines().contains(Engine.MANUAL));
     }
 
 
@@ -3088,8 +3095,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                   .legalBasis("Article legal basis")
                                                                                                   .section("section")
                                                                                                   .value("Luke Skywalker")
-                                                                                                  .build()),
-                                                                                   false).getManualAnnotationResponses()
+                                                                                                  .build())).getManualAnnotationResponses()
                 .get(0);
 
         assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
@@ -3135,7 +3141,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
         when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
 
-        ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false);
+        ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels);
         assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 600);
     }
 
@@ -3209,7 +3215,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
         when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
 
-        ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequestModels, false);
+        ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequestModels);
         assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 600);
     }
 
@@ -3241,7 +3247,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
         when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
 
-        var result = assertThrows(FeignException.class, () -> manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false));
+        var result = assertThrows(FeignException.class, () -> manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels));
         assertTrue(result.getMessage().contains("Maximum number of remove from dictionary requests is 100."));
     }
 
@@ -3353,8 +3359,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                               .section(otherSection)
                                                                                                               .value("Luke Skywalker37")
                                                                                                               .comment(comment1)
-                                                                                                              .build(),
-                                                                                                      false);
+                                                                                                              .build());
         assertEquals(1, manualRedactionResponse.getManualAnnotationResponses().size());
         assertEquals(newType.getType(),
                      manualRedactionResponse.getManualAnnotationResponses()
@@ -3383,8 +3388,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                       .value(darthVader)
                                                                                       .originLegalBases(Set.of(legal3))
                                                                                       .comment(comment2)
-                                                                                      .build(),
-                                                                              false);
+                                                                                      .build());
         assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size());
         assertEquals(newType.getType(),
                      manualRedactionResponse.getManualAnnotationResponses()
@@ -3413,8 +3417,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                       .value(darthVader)
                                                                                       .originTypes(Set.of(type2.getType()))
                                                                                       .comment(comment3)
-                                                                                      .build(),
-                                                                              false);
+                                                                                      .build());
         assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size());
         assertEquals(newType.getType(),
                      manualRedactionResponse.getManualAnnotationResponses()
@@ -3439,8 +3442,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                          .rectangle(false)
                                                                                          .value(darthVader)
                                                                                          .comment(comment4)
-                                                                                         .build(),
-                                                                                 false);
+                                                                                         .build());
         assertEquals(202, manualRedactionResponse.getManualAnnotationResponses().size());
         Long commentId4 = manualRedactionResponse.getManualAnnotationResponses()
                 .get(0).getCommentId();
@@ -3498,7 +3500,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .position(filterPosition)
                 .build();
 
-        ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), recategorizationRequest, false);
+        ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), recategorizationRequest);
 
         assertNotNull(manualRedactionResponse);
         assertEquals(3, manualRedactionResponse.getManualAnnotationResponses().size());
@@ -3518,7 +3520,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
         RecategorizationBulkLocalRequestModel finalRecategorizationRequest = recategorizationRequest;
         FeignException feignException = assertThrows(FeignException.class,
-                                                     () -> manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), finalRecategorizationRequest, false));
+                                                     () -> manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), finalRecategorizationRequest));
 
         assertEquals(feignException.status(), 400);
 
@@ -3530,12 +3532,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .position(filterPosition)
                 .build();
 
-        feignException = assertThrows(FeignException.class, () -> manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), finalRecategorizationRequest, false));
+        feignException = assertThrows(FeignException.class, () -> manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), finalRecategorizationRequest));
         assertEquals(feignException.status(), 400);
 
         RemoveRedactionBulkLocalRequestModel removeRequest = RemoveRedactionBulkLocalRequestModel.builder().rectangle(true).position(filterPosition).build();
 
-        manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRequest, false);
+        manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRequest);
 
         assertNotNull(manualRedactionResponse);
         assertEquals(3, manualRedactionResponse.getManualAnnotationResponses().size());
@@ -3550,7 +3552,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         removeRequest = RemoveRedactionBulkLocalRequestModel.builder().rectangle(false).position(filterPosition).build();
 
         RemoveRedactionBulkLocalRequestModel finalRemoveRequest = removeRequest;
-        assertThrows(FeignException.class, () -> manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), finalRemoveRequest, false));
+        assertThrows(FeignException.class, () -> manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), finalRemoveRequest));
         assertEquals(feignException.status(), 400);
     }
 
@@ -3642,8 +3644,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                             .section(newSection)
                                                                                                             .value(valueMixedCase)
                                                                                                             .caseSensitive(true)
-                                                                                                            .build(),
-                                                                                                    false);
+                                                                                                            .build());
 
         assertEquals(10, responseCaseSensitive.getManualAnnotationResponses().size());
         for (ManualAnnotationResponse response : responseCaseSensitive.getManualAnnotationResponses()) {
@@ -3662,8 +3663,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                               .section(newSection)
                                                                                                               .value(valueMixedCase)
                                                                                                               .caseSensitive(false)
-                                                                                                              .build(),
-                                                                                                      false);
+                                                                                                              .build());
 
         assertEquals(30, responseCaseInsensitive.getManualAnnotationResponses().size());
         for (ManualAnnotationResponse response : responseCaseInsensitive.getManualAnnotationResponses()) {
@@ -3679,8 +3679,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                                       .rectangle(false)
                                                                                                                       .value(valueMixedCase)
                                                                                                                       .caseSensitive(true)
-                                                                                                                      .build(),
-                                                                                                              false);
+                                                                                                                      .build());
 
         assertEquals(10, removalResponseCaseSensitive.getManualAnnotationResponses().size());
 
@@ -3690,10 +3689,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                                                                                         .rectangle(false)
                                                                                                                         .value(valueMixedCase)
                                                                                                                         .caseSensitive(false)
-                                                                                                                        .build(),
-                                                                                                                false);
+                                                                                                                        .build());
 
-        assertEquals(30, removalResponseCaseInsensitive.getManualAnnotationResponses().size());
+        assertEquals(20, removalResponseCaseInsensitive.getManualAnnotationResponses().size()); // 10 have already been removed by the previous bulk remove
     }
 
 
@@ -3753,7 +3751,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
         RemoveRedactionBulkLocalRequestModel removeValueRequest = RemoveRedactionBulkLocalRequestModel.builder().rectangle(false).value(value).build();
 
-        ManualRedactionResponse removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest, false);
+        ManualRedactionResponse removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest);
 
         assertEquals(1, removeValueResponse.getManualAnnotationResponses().size());
         EntityLog currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId());
@@ -3811,7 +3809,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         assertEquals(commentEntities.get(0).getText(), awesomeComment);
         assertEquals(commentEntities.get(0).getUser(), myUser);
 
-        removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest, true);
+        removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest);
 
         assertEquals(1, removeValueResponse.getManualAnnotationResponses().size());
         currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId());
@@ -3861,7 +3859,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .comment(totallyDifferentComment)
                 .build();
 
-        ManualRedactionResponse removeRectangleResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRectangleRequest, false);
+        ManualRedactionResponse removeRectangleResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRectangleRequest);
 
         assertEquals(1, removeRectangleResponse.getManualAnnotationResponses().size());
         Long commentId = removeRectangleResponse.getManualAnnotationResponses()
@@ -3909,7 +3907,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .orElse(null);
         assertNotNull(newRectangleAnnotation);
 
-        removeRectangleResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRectangleRequest, true);
+        removeRectangleResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRectangleRequest);
 
         assertEquals(1, removeRectangleResponse.getManualAnnotationResponses().size());
         currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId());
@@ -4030,4 +4028,148 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         when(entityLogService.getEntityLog(any(), any())).thenAnswer(answer);
     }
 
+
+    @Test
+    public void testReaddSameImageRecatsAfterFullFileOverwrite() {
+
+        // Step 1: Prepare file
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
+        var file = fileTesterAndProvider.testAndProvideFile(dossier);
+
+        // Step 2: Do recategorizations
+        var type1 = typeProvider.testAndProvideType(dossierTemplate, dossier, "Type1", false, 66);
+        var type2 = typeProvider.testAndProvideType(dossierTemplate, dossier, "Type2", false, 100);
+
+        List<String> entries = List.of("Luke Skywalker");
+        dictionaryClient.addEntry(type1.getType(), type1.getDossierTemplateId(), entries, false, null, DictionaryEntryType.ENTRY);
+
+        // Create an annotation
+        var annotationId = "AnnotationId";
+        var entityLog = new EntityLog(1,
+                                      1,
+                                      List.of(EntityLogEntry.builder()
+                                                      .id(annotationId)
+                                                      .type(type1.getType())
+                                                      .value("Luke Skywalker")
+                                                      .dictionaryEntry(true)
+                                                      .entryType(EntryType.IMAGE)
+                                                      .state(EntryState.APPLIED)
+                                                      .build()),
+                                      null,
+                                      0,
+                                      0,
+                                      0,
+                                      0);
+        fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
+
+        when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
+
+        Set<RecategorizationRequestModel> recategorizationRequests = Set.of(RecategorizationRequestModel.builder()
+                                                                                    .type(type2.getType())
+                                                                                    .annotationId(annotationId)
+                                                                                    .addToDictionary(false)
+                                                                                    .addToAllDossiers(false)
+                                                                                    .legalBasis("")
+                                                                                    .section("section")
+                                                                                    .value("Luke Skywalker")
+                                                                                    .build());
+        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequests);
+
+        var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
+        assertEquals(allManualRedactions.getRecategorizations().size(), 1);
+
+        List<ManualRecategorizationEntity> recats = recategorizationPersistenceService.findEntriesByFileIdAndOptions(file.getId(),
+                                                                                                                     ManualChangesQueryOptions.builder()
+                                                                                                                             .includeDeletions(true)
+                                                                                                                             .build());
+        assertEquals(recats.size(), 1);
+        assertEquals(annotationId, recats.get(0).getId().getAnnotationId());
+        assertNull(recats.get(0).getSoftDeletedTime());
+
+        // Step 3: file overwrite
+        fileStatusService.overwriteFile(file.getDossierId(), file.getId(), "testUploader", file.getFilename(), false, false);
+
+        fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
+
+        allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
+        assertEquals(allManualRedactions.getRecategorizations().size(), 0);
+        recats = recategorizationPersistenceService.findEntriesByFileIdAndOptions(file.getId(), ManualChangesQueryOptions.builder().includeDeletions(true).build());
+        assertEquals(recats.size(), 1);
+        assertEquals(annotationId, recats.get(0).getId().getAnnotationId());
+        assertNotNull(recats.get(0).getSoftDeletedTime());
+
+        // Step 4: Do the same recategorizations again
+        manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequests);
+
+        allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
+        recats = recategorizationPersistenceService.findEntriesByFileIdAndOptions(file.getId(), ManualChangesQueryOptions.builder().includeDeletions(true).build());
+        assertEquals(recats.size(), 1);
+        assertEquals(annotationId, recats.get(0).getId().getAnnotationId());
+        assertNull(recats.get(0).getSoftDeletedTime());
+        assertEquals(allManualRedactions.getRecategorizations().size(), 1);
+    }
+
+    @Test
+    public void testPendingEntryFactoryEngineBasedOnAddToDossierDictionary() {
+
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
+        var file = fileTesterAndProvider.testAndProvideFile(dossier);
+
+        var type = typeProvider.testAndProvideType(dossierTemplate, null, "typeEngineTest", false);
+        assertThat(type.isDossierDictionaryOnly()).isFalse();
+
+        AddRedactionRequestModel addRedactionAllDossiers = AddRedactionRequestModel.builder()
+                .positions(List.of(Rectangle.builder().topLeftX(10).topLeftY(10).width(50).height(50).page(1).build()))
+                .section("Test Section")
+                .addToDictionary(true)
+                .addToAllDossiers(true)
+                .type(type.getType())
+                .reason("Adding to all dossiers")
+                .value("TestValueAllDossiers")
+                .legalBasis("Legal Basis")
+                .sourceId("SourceId1")
+                .build();
+
+        manualRedactionClient.addRedactionBulk(dossier.getId(), file.getId(), Set.of(addRedactionAllDossiers));
+
+        AddRedactionRequestModel addRedactionSingleDossier = AddRedactionRequestModel.builder()
+                .positions(List.of(Rectangle.builder().topLeftX(20).topLeftY(20).width(30).height(30).page(2).build()))
+                .section("Test Section Single Dossier")
+                .addToDictionary(true)
+                .addToAllDossiers(false)
+                .type(type.getType())
+                .reason("Adding to single dossier")
+                .value("TestValueSingleDossier")
+                .legalBasis("Legal Basis")
+                .sourceId("SourceId2")
+                .build();
+
+        manualRedactionClient.addRedactionBulk(dossier.getId(), file.getId(), Set.of(addRedactionSingleDossier));
+
+        var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true);
+
+        DossierEntity dossierEntity = MagicConverter.convert(dossier, DossierEntity.class);
+        EntityLog updatedEntityLog = entityLogMergeService.mergeEntityLog(
+                allManualRedactions,
+                new EntityLog(),
+                dossierEntity
+        );
+
+        var entityLogEntries = updatedEntityLog.getEntityLogEntry();
+        assertThat(entityLogEntries).hasSize(2);
+
+        for (EntityLogEntry entry : entityLogEntries) {
+            if (entry.getValue().equals("TestValueAllDossiers")) {
+                // Should have Engine.DICTIONARY
+                assertThat(entry.getEngines()).containsExactlyInAnyOrder(Engine.DICTIONARY);
+            } else if (entry.getValue().equals("TestValueSingleDossier")) {
+                // Should have Engine.DOSSIER_DICTIONARY
+                assertThat(entry.getEngines()).containsExactlyInAnyOrder(Engine.DOSSIER_DICTIONARY);
+            } else {
+                throw new AssertionError("Unexpected EntityLogEntry value: " + entry.getValue());
+            }
+        }
+    }
 }

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java

@@ -645,7 +645,10 @@ public abstract class AbstractPersistenceServerServiceTest {
         public InMemoryUserDetailsManager userDetailsService(PasswordEncoder passwordEncoder) {
 
             var allRoles = getAllRoles();
-            UserDetails user = User.withUsername("manageradmin1@test.com").password(passwordEncoder.encode("secret")).roles(allRoles).authorities(getAllRoles()).build();
+            UserDetails user1 = User.withUsername("manageradmin1@test.com")
+                    .password(passwordEncoder.encode("secret")).roles(allRoles).authorities(getAllRoles()).build();
+            UserDetails user2 = User.withUsername("manageradmin2@test.com")
+                    .password(passwordEncoder.encode("secret")).roles(allRoles).authorities(getAllRoles()).build();
             var allRolesWithoutRedUserOrRedManager = Arrays.stream(allRoles)
                     .filter(s -> !(s.equalsIgnoreCase(ApplicationRoles.RED_USER_ROLE) || s.equalsIgnoreCase(ApplicationRoles.RED_MANAGER_ROLE)))
                     .collect(Collectors.toList());
@@ -654,7 +657,7 @@ public abstract class AbstractPersistenceServerServiceTest {
                     .roles(allRolesWithoutRedUserOrRedManager.toArray(new String[0]))
                     .authorities(getAllRoles())
                     .build();
-            return new InMemoryUserDetailsManager(user, userWithoutRedUserOrRedManager);
+            return new InMemoryUserDetailsManager(user1,user2, userWithoutRedUserOrRedManager);
         }
 
 

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierChangeEntry.java

@@ -5,7 +5,7 @@ import lombok.Data;
 
 @Data
 @AllArgsConstructor
-public class DossierChangeEntry {
+public class DossierChangeEntry implements IHavingDossierId {
 
     private String dossierId;
     private boolean dossierChanges;

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierChangeEntryV2.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.persistence.service.v1.api.shared.model;
+
+import java.time.OffsetDateTime;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class DossierChangeEntryV2 implements IHavingDossierId {
+
+    private String dossierId;
+    private OffsetDateTime lastUpdated;
+
+}

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierChangeResponseV2.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.persistence.service.v1.api.shared.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class DossierChangeResponseV2 {
+
+    private List<DossierChangeEntryV2> dossierChanges = new ArrayList<>();
+    private List<FileChangeEntryV2> fileChanges = new ArrayList<>();
+
+}

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileChangeEntryV2.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.persistence.service.v1.api.shared.model;
+
+import java.time.OffsetDateTime;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class FileChangeEntryV2 implements IHavingDossierId {
+
+    private String fileId;
+    private String dossierId;
+    private OffsetDateTime lastUpdated;
+
+}

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/IHavingDossierId.java

@@ -0,0 +1,6 @@
+package com.iqser.red.service.persistence.service.v1.api.shared.model;
+
+public interface IHavingDossierId {
+
+    String getDossierId();
+}

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ForceRedactionRequest.java

@@ -16,6 +16,5 @@ public class ForceRedactionRequest {
     private String legalBasis;
     private String comment;
     private int page;
-    private String basedOnDictAnnotationId;
 
 }