Merge branch 'RE-10691-fp' into 'master'

RED-10691: 500 when approving file in a dossier where dossier owner lost manager role

Closes RE-10691

See merge request redactmanager/persistence-service!930

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -7,9 +7,10 @@ plugins {
 }
 
 val redactionServiceVersion by rootProject.extra { "4.290.0" }
-val pdftronRedactionServiceVersion by rootProject.extra { "4.87.0" }
+val pdftronRedactionServiceVersion by rootProject.extra { "4.90.0-RED10115.0" }
 val redactionReportServiceVersion by rootProject.extra { "4.81.0" }
 val searchServiceVersion by rootProject.extra { "2.90.0" }
+val documentVersion by rootProject.extra { "4.433.0" }
 
 repositories {
     mavenLocal()

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DictionaryController.java

@@ -357,9 +357,9 @@ public class DictionaryController implements DictionaryResource {
     public void changeFlags(@PathVariable(TYPE_PARAMETER_NAME) String type,
                             @PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
                             @PathVariable(value = DOSSIER_ID_PARAMETER_NAME) String dossierId,
-                            @RequestParam(value = "addToDictionary") boolean addToDictionary) {
+                            @RequestParam(value = "addToDictionaryAction") boolean addToDictionaryAction) {
 
-        dictionaryService.changeAddToDictionary(type, dossierTemplateId, dossierId, addToDictionary);
+        dictionaryService.changeAddToDictionary(type, dossierTemplateId, dossierId, addToDictionaryAction);
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierAttributesController.java

@@ -17,6 +17,7 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.DossierAttributeConfigMapper;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
@@ -52,7 +53,7 @@ public class DossierAttributesController implements DossierAttributesResource {
         var result = MagicConverter.convert(dossierAttributeConfigPersistenceService.setDossierAttributesConfig(dossierTemplateId,
                                                                                                                 MagicConverter.convert(dossierAttributesConfig.getDossierAttributeConfigs(),
                                                                                                                                        DossierAttributeConfigEntity.class)),
-                                            DossierAttributeConfig.class);
+                                            DossierAttributeConfig.class, new DossierAttributeConfigMapper());
         auditPersistenceService.insertRecord(AuditRequest.builder()
                                                      .userId(KeycloakSecurity.getUserId())
                                                      .objectId(dossierTemplateId)
@@ -72,7 +73,7 @@ public class DossierAttributesController implements DossierAttributesResource {
         var result = MagicConverter.convert(dossierAttributeConfigPersistenceService.addOrUpdateDossierAttribute(dossierTemplateId,
                                                                                                                  MagicConverter.convert(dossierAttribute,
                                                                                                                                         DossierAttributeConfigEntity.class)),
-                                            DossierAttributeConfig.class);
+                                            DossierAttributeConfig.class, new DossierAttributeConfigMapper());
         auditPersistenceService.insertRecord(AuditRequest.builder()
                                                      .userId(KeycloakSecurity.getUserId())
                                                      .objectId(dossierTemplateId)
@@ -118,7 +119,7 @@ public class DossierAttributesController implements DossierAttributesResource {
     public DossierAttributesConfig getDossierAttributesConfig(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
 
         var result = dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId);
-        return new DossierAttributesConfig(MagicConverter.convert(result, DossierAttributeConfig.class));
+        return new DossierAttributesConfig(MagicConverter.convert(result, DossierAttributeConfig.class, new DossierAttributeConfigMapper()));
     }
 
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java

@@ -11,6 +11,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -23,7 +24,11 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.model.websocket.DossierEventType;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
 
+import com.iqser.red.service.persistence.management.v1.processor.service.FilterByPermissionsService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.JsonNode;
 import org.apache.commons.lang3.StringUtils;
+
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -76,6 +81,7 @@ public class DossierController implements DossierResource {
 
     private final DossierManagementService dossierManagementService;
     private final UserService userService;
+    private final FilterByPermissionsService filterByPermissionsService;
     private final FileStatusManagementService fileStatusManagementService;
     private final AuditPersistenceService auditPersistenceService;
     private final NotificationPersistenceService notificationPersistenceService;
@@ -106,6 +112,20 @@ public class DossierController implements DossierResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
+    public DossierChangeResponseV2 changesSinceV2(@RequestBody JSONPrimitive<OffsetDateTime> since) {
+
+        DossierChangeResponseV2 changes = dossierManagementService.changesSinceV2(since);
+
+        // filter only viewables
+        changes.setFileChanges(filterByPermissionsService.onlyViewableHavingDossierId(changes.getFileChanges()));
+        changes.setDossierChanges(filterByPermissionsService.onlyViewableHavingDossierId(changes.getDossierChanges()));
+
+        return changes;
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DOSSIER + "') && (#dossierRequest.dossierId == null || hasPermission(#dossierRequest.dossierId, 'Dossier', 'ACCESS_OBJECT') )")
     public ResponseEntity<Dossier> createDossierOrUpdateDossier(@RequestBody DossierRequest dossierRequest) {
@@ -405,6 +425,28 @@ public class DossierController implements DossierResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
+    public JSONPrimitive<Map<String, Dossier>> getDossiersByIds(@RequestBody JSONPrimitive<Set<String>> dossierIds) {
+
+        // filter dossiers based on view
+        var viewableDossierIds = filterByPermissionsService.onlyViewableDossierIds(dossierIds.getValue());
+        // load dossiers
+        var dossiers = dossierManagementService.getDossiersByIds(viewableDossierIds);
+
+        // add attributes and ACL - already filtered before loading
+        enhanceDossiersWithAttributeAndACLData(dossiers,false);
+
+        // build response
+        var responseMap = new LinkedHashMap<String, Dossier>();
+        for (var dossier : dossiers) {
+            responseMap.put(dossier.getId(), dossier);
+        }
+
+        return new JSONPrimitive<>(responseMap);
+    }
+
+
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
     public Dossier getDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
                               @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
@@ -418,70 +460,45 @@ public class DossierController implements DossierResource {
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                                      @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted) {
 
-        var dossiers = dossierManagementService.getAllDossiers(includeArchived, includeDeleted)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getAllDossiers(includeArchived, includeDeleted);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getDossiersForDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
                                                        @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                                                        @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted) {
 
-        var dossiers = dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getSoftDeletedDossiers() {
 
-        var dossiers =  dossierManagementService.getSoftDeletedDossiers()
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getSoftDeletedDossiers();
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiers() {
 
-        var dossiers =  dossierManagementService.getArchivedDossiers()
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getArchivedDossiers();
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
     @PreAuthorize("hasAuthority('" + READ_DOSSIER + "')")
-    @PostFilter("hasPermission(filterObject.id, 'Dossier', 'VIEW_OBJECT')")
     public List<Dossier> getArchivedDossiersForDossierTemplate(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId) {
 
-        var dossiers = dossierManagementService.getArchivedDossiersForDossierTemplateId(dossierTemplateId)
-                .stream()
-                .map(dossierACLService::enhanceDossierWithACLData)
-                .collect(Collectors.toList());
-        dossiers.forEach(dossier -> dossier.setDossierAttributes(convertDossierAttributes(dossierAttributePersistenceService.getDossierAttributes(dossier.getId()))));
-        return dossiers;
+        var dossiers = dossierManagementService.getArchivedDossiersForDossierTemplateId(dossierTemplateId);
+        return enhanceDossiersWithAttributeAndACLData(dossiers);
     }
 
 
@@ -586,5 +603,31 @@ public class DossierController implements DossierResource {
         return new DossierAttributes(attributeIdToValue);
     }
 
+    private List<Dossier> enhanceDossiersWithAttributeAndACLData(List<Dossier> dossiers) {
+
+            return enhanceDossiersWithAttributeAndACLData(dossiers, true);
+    }
+
+    private List<Dossier> enhanceDossiersWithAttributeAndACLData(List<Dossier> dossiers, boolean filter) {
+
+        // filter first, only load attributes and ACL for viewable dossiers
+        List<Dossier> filteredDossiers = filter ? filterByPermissionsService.onlyViewableDossiers(dossiers) : dossiers;
+
+        // load all attributes at once
+        var attributes = dossierAttributePersistenceService.getDossierAttributes(filteredDossiers.stream().map(Dossier::getId).collect(Collectors.toSet()));
+        var attributesMap = new HashMap<String, List<DossierAttributeEntity>>();
+        for (DossierAttributeEntity attribute : attributes) {
+            attributesMap.computeIfAbsent(attribute.getId().getDossierId(), k -> new ArrayList<>()).add(attribute);
+        }
+
+        for (var dossier : filteredDossiers) {
+            // set attributes
+            dossier.setDossierAttributes(convertDossierAttributes(attributesMap.getOrDefault(dossier.getId(), new ArrayList<>())));
+            // set ACL data
+            dossierACLService.enhanceDossierWithACLData(dossier);
+        }
+        return filteredDossiers;
+    }
+
 }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierStatusController.java

@@ -67,7 +67,7 @@ public class DossierStatusController implements DossierStatusResource {
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierStatusRequest.getDossierTemplateId())
                                               .category(AuditCategory.DOSSIER_TEMPLATE.name())
-                                              .message("Dossier states has been updated.")
+                                              .message("Dossier states have been updated.")
                                               .build());
 
 
@@ -105,7 +105,17 @@ public class DossierStatusController implements DossierStatusResource {
     public void deleteDossierStatus(@PathVariable("dossierStatusId") String dossierStatusId,
                                     @RequestParam(value = DOSSIER_STATUS_REPLACE_ID, required = false) String replaceDossierStatusId) {
 
+        var dossierTemplateId = dossierStatusPersistenceService.getDossierStatus(dossierStatusId).getDossierTemplateId();
+
         dossierStatusPersistenceService.deleteDossierStatus(dossierStatusId, replaceDossierStatusId);
+
+        auditPersistenceService.audit(AuditRequest.builder()
+                                              .userId(KeycloakSecurity.getUserId())
+                                              .objectId(dossierTemplateId)
+                                              .category(AuditCategory.DOSSIER_TEMPLATE.name())
+                                              .message("Dossier state has been deleted.")
+                                              .build());
+
     }
 
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DownloadController.java

@@ -7,23 +7,17 @@ import java.io.BufferedInputStream;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.InputStreamResource;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.context.request.RequestAttributes;
 import org.springframework.web.context.request.RequestContextHolder;
 import org.springframework.web.context.request.ServletRequestAttributes;
 
@@ -34,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.AccessC
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DownloadService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
@@ -76,6 +71,7 @@ public class DownloadController implements DownloadResource {
     private final OneTimeTokenService oneTimeTokenDownloadService;
     private final AccessControlService accessControlService;
     private final FileManagementStorageService fileManagementStorageService;
+    private final FileStatusManagementService fileStatusManagementService;
 
     private final String REPORT_INFO = "/REPORT_INFO.json";
 
@@ -142,7 +138,7 @@ public class DownloadController implements DownloadResource {
         if (StringUtils.isBlank(dossierId)) {
             throw new BadRequestException("Empty dossier id");
         }
-        dossierService.getDossierById(dossierId, true, true);
+        dossierService.getDossierById(dossierId, true, false);
         accessControlService.verifyUserIsDossierOwnerOrApprover(dossierId);
     }
 
@@ -152,16 +148,19 @@ public class DownloadController implements DownloadResource {
         List<FileModel> validFiles = fileStatusService.getDossierStatus(request.getDossierId());
         var fileIds = request.getFileIds();
         if (fileIds != null && !fileIds.isEmpty()) { // validate the ids provided
+            if(fileIds.size() == 1) {
+                fileStatusManagementService.getFileStatus(fileIds.get(0), false);
+            }
             validFiles = validFiles.stream()
                     .filter(f -> fileIds.contains(f.getId()))
                     .collect(Collectors.toList());
             if (validFiles.isEmpty()) {
-                throw new NotFoundException("No file id provided is found");
+                throw new NotFoundException("No provided file id was found");
             }
         } // otherwise consider the files from dossier
 
         var validFilesAndNotProcessed = validFiles.stream()
-                .filter(f -> !(f.getAnalysisVersion() > 0 && f.getNumberOfAnalyses() > 0))
+                .filter(f -> !(f.getAnalysisVersion() > 0 && f.getNumberOfAnalyses() > 0 && !f.isSoftOrHardDeleted()))
                 .collect(Collectors.toList());
         if (!validFilesAndNotProcessed.isEmpty()) {
             throw new BadRequestException("At least a file is in its initial analysis process");

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileAttributesController.java

@@ -15,6 +15,7 @@ import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
 import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileAttributeConfigMapper;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
@@ -61,9 +62,9 @@ public class FileAttributesController implements FileAttributesResource {
         }
         fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(dossierTemplateId,
                                                                              MagicConverter.convert(fileAttributesConfig, FileAttributesGeneralConfigurationEntity.class));
-        var result = fileAttributeConfigPersistenceService.setFileAttributesConfig(dossierTemplateId,
-                                                                                   MagicConverter.convert(fileAttributesConfig.getFileAttributeConfigs(),
-                                                                                                          FileAttributeConfigEntity.class));
+        fileAttributeConfigPersistenceService.setFileAttributesConfig(dossierTemplateId,
+                                                                      MagicConverter.convert(fileAttributesConfig.getFileAttributeConfigs(), FileAttributeConfigEntity.class));
+
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierTemplateId)
@@ -74,7 +75,9 @@ public class FileAttributesController implements FileAttributesResource {
                 .filenameMappingColumnHeaderName(fileAttributesConfig.getFilenameMappingColumnHeaderName())
                 .delimiter(fileAttributesConfig.getDelimiter())
                 .encoding(fileAttributesConfig.getEncoding())
-                .fileAttributeConfigs(MagicConverter.convert(result, FileAttributeConfig.class))
+                .fileAttributeConfigs(MagicConverter.convert(fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId),
+                                                             FileAttributeConfig.class,
+                                                             new FileAttributeConfigMapper()))
                 .build();
 
     }
@@ -96,7 +99,7 @@ public class FileAttributesController implements FileAttributesResource {
                                                               dossierTemplateId))
                                               .build());
 
-        return MagicConverter.convert(result, FileAttributeConfig.class);
+        return MagicConverter.convert(result, FileAttributeConfig.class, new FileAttributeConfigMapper());
     }
 
 
@@ -145,7 +148,7 @@ public class FileAttributesController implements FileAttributesResource {
                 .filenameMappingColumnHeaderName(generalConfig.getFilenameMappingColumnHeaderName())
                 .delimiter(generalConfig.getDelimiter())
                 .encoding(generalConfig.getEncoding())
-                .fileAttributeConfigs(MagicConverter.convert(fileAttributeConfigs, FileAttributeConfig.class))
+                .fileAttributeConfigs(MagicConverter.convert(fileAttributeConfigs, FileAttributeConfig.class, new FileAttributeConfigMapper()))
                 .build();
     }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java

@@ -232,7 +232,7 @@ public class FileManagementController implements FileManagementResource {
     public void restoreFiles(@PathVariable(DOSSIER_ID) String dossierId, @RequestBody Set<String> fileIds) {
 
         accessControlService.checkAccessPermissionsToDossier(dossierId);
-        verifyUserIsDossierOwnerOrApproverOrAssignedReviewer(dossierId, fileIds);
+        accessControlService.verifyUserIsDossierOwnerOrApproverOrAssignedReviewer(dossierId, fileIds);
         fileService.undeleteFiles(dossierId, fileIds);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
@@ -289,20 +289,4 @@ public class FileManagementController implements FileManagementResource {
         }
     }
 
-
-    private void verifyUserIsDossierOwnerOrApproverOrAssignedReviewer(String dossierId, Set<String> fileIds) {
-
-        try {
-            accessControlService.verifyUserIsDossierOwnerOrApprover(dossierId);
-        } catch (AccessDeniedException e1) {
-            try {
-                for (String fileId : fileIds) {
-                    accessControlService.verifyUserIsReviewer(dossierId, fileId);
-                }
-            } catch (NotAllowedException e2) {
-                throw new NotAllowedException("User must be dossier owner, approver or assigned reviewer of the file.");
-            }
-        }
-    }
-
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java

@@ -8,6 +8,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.roles.Ac
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -20,6 +21,7 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.mapper.EntityLogResponseMapper;
 import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
 import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
@@ -90,6 +92,7 @@ public class ManualRedactionController implements ManualRedactionResource {
     DictionaryPersistenceService dictionaryPersistenceService;
 
     EntityLogController entityLogController;
+    EntityLogResponseMapper mapper = EntityLogResponseMapper.INSTANCE;
 
 
     @PreAuthorize("hasAuthority('" + DELETE_MANUAL_REDACTION + "')")
@@ -260,11 +263,7 @@ public class ManualRedactionController implements ManualRedactionResource {
                                removeRedactionRequests.stream()
                                        .anyMatch(RemoveRedactionRequestModel::isRemoveFromAllDossiers));
 
-        List<ManualAnnotationResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId,
-                                                                                                fileId,
-                                                                                                removeRedactionRequests,
-                                                                                                dossier.getDossierTemplateId(),
-                                                                                                true);
+        List<ManualAnnotationResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId, fileId, removeRedactionRequests, dossier.getDossierTemplateId(), true);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
                                                                                .userId(KeycloakSecurity.getUserId())
@@ -279,9 +278,7 @@ public class ManualRedactionController implements ManualRedactionResource {
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
-    public ManualRedactionResponse removeRedactionBulkLocal(String dossierId,
-                                                            String fileId,
-                                                            RemoveRedactionBulkLocalRequestModel removeRedactionRequest) {
+    public ManualRedactionResponse removeRedactionBulkLocal(String dossierId, String fileId, RemoveRedactionBulkLocalRequestModel removeRedactionRequest) {
 
         verifyAccess(dossierId, fileId);
         verifyRequest(removeRedactionRequest.isRectangle(), removeRedactionRequest.getPosition(), removeRedactionRequest.getValue());
@@ -289,37 +286,35 @@ public class ManualRedactionController implements ManualRedactionResource {
         Set<RemoveRedactionRequestModel> removeRedactionRequestModels;
         FileModel status = fileStatusService.getStatus(fileId);
 
+        Set<EntityLogEntry> entries;
         if (!status.isExcludedFromAutomaticAnalysis()) {
-            Set<EntityLogEntry> entries = getFilteredEntityLogEntries(dossierId,
-                                                                      fileId,
-                                                                      removeRedactionRequest.isRectangle(),
-                                                                      removeRedactionRequest.getValue(),
-                                                                      removeRedactionRequest.isCaseSensitive(),
-                                                                      removeRedactionRequest.getOriginTypes(),
-                                                                      removeRedactionRequest.getOriginLegalBases(),
-                                                                      removeRedactionRequest.getPageNumbers(),
-                                                                      removeRedactionRequest.getPosition());
-            removeRedactionRequestModels = entries.stream()
-                    .map(entry -> RemoveRedactionRequestModel.builder().annotationId(entry.getId()).comment(removeRedactionRequest.getComment()).build())
-                    .collect(Collectors.toSet());
+            entries = getFilteredEntityLogEntries(dossierId,
+                                                  fileId,
+                                                  removeRedactionRequest.isRectangle(),
+                                                  removeRedactionRequest.getValue(),
+                                                  removeRedactionRequest.isCaseSensitive(),
+                                                  removeRedactionRequest.getOriginTypes(),
+                                                  removeRedactionRequest.getOriginLegalBases(),
+                                                  removeRedactionRequest.getPageNumbers(),
+                                                  removeRedactionRequest.getPosition());
 
         } else {
 
-            List<EntityLogEntryResponse> filteredEntityLogResponses = getFilteredEntityLogResponses(dossierId,
-                                                                                                    fileId,
-                                                                                                    true,
-                                                                                                    removeRedactionRequest.isRectangle(),
-                                                                                                    removeRedactionRequest.getValue(),
-                                                                                                    removeRedactionRequest.isCaseSensitive(),
-                                                                                                    removeRedactionRequest.getOriginTypes(),
-                                                                                                    removeRedactionRequest.getOriginLegalBases(),
-                                                                                                    removeRedactionRequest.getPageNumbers(),
-                                                                                                    removeRedactionRequest.getPosition());
-
-            removeRedactionRequestModels = filteredEntityLogResponses.stream()
-                    .map(entityLogEntry -> RemoveRedactionRequestModel.builder().annotationId(entityLogEntry.getId()).comment(removeRedactionRequest.getComment()).build())
-                    .collect(Collectors.toSet());
+            entries = new HashSet<>(mapper.fromLogEntryResponses(getFilteredEntityLogResponses(dossierId,
+                                                                                               fileId,
+                                                                                               removeRedactionRequest.isRectangle(),
+                                                                                               removeRedactionRequest.getValue(),
+                                                                                               removeRedactionRequest.isCaseSensitive(),
+                                                                                               removeRedactionRequest.getOriginTypes(),
+                                                                                               removeRedactionRequest.getOriginLegalBases(),
+                                                                                               removeRedactionRequest.getPageNumbers(),
+                                                                                               removeRedactionRequest.getPosition())));
         }
+
+        removeRedactionRequestModels = entries.stream()
+                .map(entityLogEntry -> RemoveRedactionRequestModel.builder().annotationId(entityLogEntry.getId()).comment(removeRedactionRequest.getComment()).build())
+                .collect(Collectors.toSet());
+
         return removeRedactionBulk(dossierId, fileId, removeRedactionRequestModels);
     }
 
@@ -389,62 +384,49 @@ public class ManualRedactionController implements ManualRedactionResource {
 
 
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
-    public ManualRedactionResponse recategorizeBulkLocal(String dossierId,
-                                                         String fileId,
-                                                         RecategorizationBulkLocalRequestModel recategorizationRequest) {
+    public ManualRedactionResponse recategorizeBulkLocal(String dossierId, String fileId, RecategorizationBulkLocalRequestModel recategorizationRequest) {
 
         verifyAccess(dossierId, fileId);
         verifyRequest(recategorizationRequest.isRectangle(), recategorizationRequest.getPosition(), recategorizationRequest.getValue());
         Set<RecategorizationRequestModel> recategorizationRequestModels;
         FileModel status = fileStatusService.getStatus(fileId);
 
+        Set<EntityLogEntry> entries;
         if (!status.isExcludedFromAutomaticAnalysis()) {
-            Set<EntityLogEntry> entries = getFilteredEntityLogEntries(dossierId,
-                                                                      fileId,
-                                                                      recategorizationRequest.isRectangle(),
-                                                                      recategorizationRequest.getValue(),
-                                                                      recategorizationRequest.isCaseSensitive(),
-                                                                      recategorizationRequest.getOriginTypes(),
-                                                                      recategorizationRequest.getOriginLegalBases(),
-                                                                      recategorizationRequest.getPageNumbers(),
-                                                                      recategorizationRequest.getPosition());
-
-            recategorizationRequestModels = entries.stream()
-                    .map(entry -> RecategorizationRequestModel.builder()
-                            .annotationId(entry.getId())
-                            .type(recategorizationRequest.isRectangle() ? entry.getType() : recategorizationRequest.getType())
-                            .legalBasis(recategorizationRequest.getLegalBasis())
-                            .section(recategorizationRequest.getSection())
-                            .value(entry.getValue())
-                            .comment(recategorizationRequest.getComment())
-                            .build())
-                    .collect(Collectors.toSet());
+            entries = getFilteredEntityLogEntries(dossierId,
+                                                  fileId,
+                                                  recategorizationRequest.isRectangle(),
+                                                  recategorizationRequest.getValue(),
+                                                  recategorizationRequest.isCaseSensitive(),
+                                                  recategorizationRequest.getOriginTypes(),
+                                                  recategorizationRequest.getOriginLegalBases(),
+                                                  recategorizationRequest.getPageNumbers(),
+                                                  recategorizationRequest.getPosition());
 
         } else {
 
-            List<EntityLogEntryResponse> filteredEntityLogResponses = getFilteredEntityLogResponses(dossierId,
-                                                                                                    fileId,
-                                                                                                    true,
-                                                                                                    recategorizationRequest.isRectangle(),
-                                                                                                    recategorizationRequest.getValue(),
-                                                                                                    recategorizationRequest.isCaseSensitive(),
-                                                                                                    recategorizationRequest.getOriginTypes(),
-                                                                                                    recategorizationRequest.getOriginLegalBases(),
-                                                                                                    recategorizationRequest.getPageNumbers(),
-                                                                                                    recategorizationRequest.getPosition());
-
-            recategorizationRequestModels = filteredEntityLogResponses.stream()
-                    .map(entityLogEntry -> RecategorizationRequestModel.builder()
-                            .annotationId(entityLogEntry.getId())
-                            .type(recategorizationRequest.isRectangle() ? entityLogEntry.getType() : recategorizationRequest.getType())
-                            .legalBasis(recategorizationRequest.getLegalBasis())
-                            .section(recategorizationRequest.getSection())
-                            .value(entityLogEntry.getValue())
-                            .comment(recategorizationRequest.getComment())
-                            .build())
-                    .collect(Collectors.toSet());
+            entries = new HashSet<>(mapper.fromLogEntryResponses(getFilteredEntityLogResponses(dossierId,
+                                                                                               fileId,
+                                                                                               recategorizationRequest.isRectangle(),
+                                                                                               recategorizationRequest.getValue(),
+                                                                                               recategorizationRequest.isCaseSensitive(),
+                                                                                               recategorizationRequest.getOriginTypes(),
+                                                                                               recategorizationRequest.getOriginLegalBases(),
+                                                                                               recategorizationRequest.getPageNumbers(),
+                                                                                               recategorizationRequest.getPosition())));
         }
 
+        recategorizationRequestModels = entries.stream()
+                .map(entry -> RecategorizationRequestModel.builder()
+                        .annotationId(entry.getId())
+                        .type(recategorizationRequest.isRectangle() ? entry.getType() : recategorizationRequest.getType())
+                        .legalBasis(recategorizationRequest.getLegalBasis())
+                        .section(recategorizationRequest.getSection())
+                        .value(recategorizationRequest.isRectangle() ? recategorizationRequest.getValue() : entry.getValue())
+                        .comment(recategorizationRequest.getComment())
+                        .build())
+                .collect(Collectors.toSet());
+
         return recategorizeBulk(dossierId, fileId, recategorizationRequestModels);
     }
 
@@ -492,7 +474,6 @@ public class ManualRedactionController implements ManualRedactionResource {
 
     private List<EntityLogEntryResponse> getFilteredEntityLogResponses(String dossierId,
                                                                        String fileId,
-                                                                       boolean includeUnprocessed,
                                                                        boolean rectangle,
                                                                        String value,
                                                                        boolean caseSensitive,
@@ -501,7 +482,7 @@ public class ManualRedactionController implements ManualRedactionResource {
                                                                        Set<Integer> pageNumbers,
                                                                        Position position) {
 
-        List<EntityLogEntryResponse> entityLogEntryResponses = entityLogController.getEntityLog(dossierId, fileId, Collections.emptyList(), includeUnprocessed).getEntityLogEntry()
+        List<EntityLogEntryResponse> entityLogEntryResponses = entityLogController.getEntityLog(dossierId, fileId, Collections.emptyList(), true).getEntityLogEntry()
                 .stream()
                 .filter(entityLogEntryResponse -> !entityLogEntryResponse.getState().equals(EntryState.PENDING))
                 .collect(Collectors.toList());

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java

@@ -1,111 +0,0 @@
-package com.iqser.red.persistence.service.v1.external.api.impl.controller;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
-import com.iqser.red.service.persistence.service.v1.api.external.resource.MigrationStatusResource;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*;
-
-@RestController
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@RequiredArgsConstructor
-public class MigrationStatusController implements MigrationStatusResource {
-
-    SaasMigrationService saasMigrationService;
-
-    SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
-
-    FileStatusService fileStatusService;
-
-
-    public MigrationStatusResponse migrationStatus() {
-
-        int numberOfFilesToMigrate = saasMigrationStatusPersistenceService.countAll();
-
-        Map<SaasMigrationStatus, Integer> filesInStatus = new HashMap<>();
-        filesInStatus.put(MIGRATION_REQUIRED, saasMigrationStatusPersistenceService.countByStatus(MIGRATION_REQUIRED));
-        filesInStatus.put(DOCUMENT_FILES_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(DOCUMENT_FILES_MIGRATED));
-        filesInStatus.put(REDACTION_LOGS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(REDACTION_LOGS_MIGRATED));
-        filesInStatus.put(ANNOTATION_IDS_MIGRATED, saasMigrationStatusPersistenceService.countByStatus(ANNOTATION_IDS_MIGRATED));
-        filesInStatus.put(FINISHED, saasMigrationStatusPersistenceService.countByStatus(FINISHED));
-        filesInStatus.put(ERROR, saasMigrationStatusPersistenceService.countByStatus(ERROR));
-
-        var filesInErrorState = saasMigrationStatusPersistenceService.findAllByStatus(ERROR);
-
-        var errorCauses = filesInErrorState.stream()
-                .collect(Collectors.toMap(errorFile -> errorFile.getDossierId() + "/" + errorFile.getFileId(), SaasMigrationStatusEntity::getErrorCause));
-
-        return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> startMigrationForFile(String dossierId, String fileId) {
-
-        if (!fileStatusService.fileExists(fileId)) {
-            throw new NotFoundException(String.format("File with id %s does not exist", fileId));
-        }
-
-        saasMigrationService.startMigrationForFile(dossierId, fileId);
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> revertMigrationForFile(String dossierId, String fileId) {
-
-        if (!fileStatusService.fileExists(fileId)) {
-            throw new NotFoundException(String.format("File with id %s does not exist", fileId));
-        }
-
-        if (!saasMigrationStatusPersistenceService.findById(fileId).getStatus().equals(FINISHED)) {
-            throw new BadRequestException(String.format("File with id %s is not migrated yet, can't revert.", fileId));
-        }
-
-        saasMigrationService.revertMigrationForFile(dossierId, fileId);
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    @Override
-    public ResponseEntity<?> requeueErrorFiles() {
-
-        MigrationStatusResponse migrationStatus = migrationStatus();
-        if (!migrationIsFinished(migrationStatus)) {
-            throw new BadRequestException("There are still files processing, please wait until migration has finished to retry!");
-        }
-
-        saasMigrationService.requeueErrorFiles();
-
-        return ResponseEntity.ok().build();
-    }
-
-
-    private static boolean migrationIsFinished(MigrationStatusResponse migrationStatus) {
-
-        return migrationStatus.getFilesInStatus().entrySet()
-                .stream()
-                .filter(e -> e.getValue() > 0)
-                .allMatch(e -> e.getKey().equals(FINISHED) || e.getKey().equals(ERROR));
-    }
-
-}

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java

@@ -118,11 +118,12 @@ public class ReanalysisController implements ReanalysisResource {
     @PreAuthorize("hasAuthority('" + REANALYZE_FILE + "')")
     public void ocrFile(@PathVariable(DOSSIER_ID) String dossierId,
                         @PathVariable(FILE_ID) String fileId,
-                        @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force) {
+                        @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force,
+                        @RequestParam(value = ALL_PAGES, required = false, defaultValue = FALSE) boolean allPages) {
 
         accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
         validateOCR(dossierId, fileId);
-        reanalysisService.ocrFile(dossierId, fileId, force);
+        reanalysisService.ocrFile(dossierId, fileId, force, allPages);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierId)
@@ -140,7 +141,7 @@ public class ReanalysisController implements ReanalysisResource {
 
         accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
         fileIds.forEach(fileId -> validateOCR(dossierId, fileId));
-        reanalysisService.ocrFiles(dossierId, fileIds);
+        reanalysisService.ocrFiles(dossierId, fileIds, false);
         auditPersistenceService.audit(AuditRequest.builder()
                                               .userId(KeycloakSecurity.getUserId())
                                               .objectId(dossierId)

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RulesController.java

@@ -160,4 +160,12 @@ public class RulesController implements RulesResource {
         return new ResponseEntity<>(new InputStreamResource(is), httpHeaders, HttpStatus.OK);
     }
 
+
+    @Override
+    @PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
+    public void unlockRules(String dossierTemplateId, RuleFileType ruleFileType) {
+
+        rulesPersistenceService.resetTimeoutDetected(dossierTemplateId, ruleFileType);
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java

@@ -14,15 +14,21 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
 import org.springframework.security.access.AccessDeniedException;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
 import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
@@ -30,6 +36,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Approva
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
+import com.iqser.red.service.persistence.management.v1.processor.service.FilterByPermissionsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
@@ -39,6 +46,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
@@ -46,6 +54,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.notificatio
 import com.iqser.red.service.persistence.service.v1.api.shared.model.warning.ApproveResponse;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -67,6 +78,7 @@ public class StatusController implements StatusResource {
     private final NotificationPersistenceService notificationPersistenceService;
     private final DossierACLService dossierACLService;
     private final ApprovalVerificationService approvalVerificationService;
+    private final FilterByPermissionsService filterByPermissionsService;
 
 
     @Override
@@ -82,6 +94,28 @@ public class StatusController implements StatusResource {
     }
 
 
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
+    public JSONPrimitive<Map<String, List<FileStatus>>> getFilesByIds(@RequestBody JSONPrimitive<Map<String, Set<String>>> filesByDossier) {
+
+        // filter dossiers by view
+        var accessibleDossierIds = filterByPermissionsService.onlyViewableDossierIds(new ArrayList<>(filesByDossier.getValue().keySet()));
+        var response = new HashMap<String, List<FileStatus>>();
+        for (var dossierId : accessibleDossierIds) {
+            var allFoundFiles = fileStatusManagementService.findAllDossierIdAndIds(dossierId,
+                                                                                   filesByDossier.getValue()
+                                                                                           .get(dossierId));
+            response.put(dossierId,
+                         allFoundFiles.stream()
+                                 .map(FileStatusMapper::toFileStatus)
+                                 .collect(Collectors.toList()));
+        }
+
+        return new JSONPrimitive<>(response);
+
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
     public Map<String, List<FileStatus>> getDossierStatus(@RequestBody List<String> dossierIds) {
@@ -323,6 +357,10 @@ public class StatusController implements StatusResource {
                                                   .build());
 
             var dossier = dossierACLService.enhanceDossierWithACLData(dossierManagementService.getDossierById(dossierId, false, false));
+
+            if (dossier.getOwnerId() == null) {
+                throw new ConflictException("Dossier has no owner!");
+            }
             if (!dossier.getOwnerId().equals(KeycloakSecurity.getUserId())) {
 
                 var fileStatus = fileStatusManagementService.getFileStatus(fileId);
@@ -369,8 +407,10 @@ public class StatusController implements StatusResource {
 
     private void generatePossibleUnassignedFromFileNotification(String dossierId, String fileId, FileModel oldFileStatus, String newAssigneeId) {
 
-        if (oldFileStatus.getAssignee() == null || newAssigneeId == null || oldFileStatus.getAssignee().equals(newAssigneeId) || KeycloakSecurity.getUserId()
-                .equals(oldFileStatus.getAssignee())) {
+        if (oldFileStatus.getAssignee() == null
+            || newAssigneeId == null && oldFileStatus.getAssignee() == null
+            || oldFileStatus.getAssignee().equals(newAssigneeId)
+            || KeycloakSecurity.getUserId().equals(oldFileStatus.getAssignee())) {
             return;
         }
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java

@@ -23,12 +23,11 @@ import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
-import com.iqser.red.service.persistence.management.v1.processor.service.FileFormatValidationService;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
-import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileFormatValidationService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
 import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
@@ -37,6 +36,7 @@ import com.iqser.red.service.persistence.service.v1.api.external.resource.Upload
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import feign.FeignException;
@@ -53,9 +53,9 @@ import lombok.extern.slf4j.Slf4j;
 @SuppressWarnings("PMD")
 public class UploadController implements UploadResource {
 
-    private static final int THRESHOLD_ENTRIES = 10000;
-    private static final int THRESHOLD_SIZE = 1000000000; // 1 GB
-    private static final double THRESHOLD_RATIO = 10;
+    private static final int THRESHOLD_ENTRIES = 10000;      // Maximum number of files allowed
+    private static final int THRESHOLD_SIZE = 1000000000;    // 1 GB total unzipped data
+    private static final double THRESHOLD_RATIO = 10;        // Max allowed compression ratio
 
     private final UploadService uploadService;
     private final ReanalysisService reanalysisService;
@@ -72,31 +72,25 @@ public class UploadController implements UploadResource {
                                    @Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis) {
 
         accessControlService.checkAccessPermissionsToDossier(dossierId);
-        if (file.getOriginalFilename() == null) {
+
+        String originalFilename = file.getOriginalFilename();
+        if (originalFilename == null) {
             throw new BadRequestException("Could not upload file, no filename provided.");
         }
 
-        var extension = getExtension(file.getOriginalFilename());
-
+        String extension = getExtension(originalFilename);
         try {
-            switch (extension) {
-                case "zip":
-                    return handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
-                case "csv":
-                    return uploadService.importCsv(dossierId, file.getBytes());
-                default:
-                    if (!fileFormatValidationService.getAllFileFormats().contains(extension)) {
-                        throw new BadRequestException("Invalid file uploaded");
-                    }
-                    if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
-                        throw new NotAllowedException("Insufficient permissions");
-                    }
-                    return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
-            }
+            return switch (extension) {
+                case "zip" -> handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
+                case "csv" -> uploadService.importCsv(dossierId, file.getBytes());
+                default -> {
+                    validateExtensionOrThrow(extension);
+                    yield uploadService.processSingleFile(dossierId, originalFilename, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
+                }
+            };
         } catch (IOException e) {
-            throw new BadRequestException(e.getMessage(), e);
+            throw new BadRequestException("Failed to process file: " + e.getMessage(), e);
         }
-
     }
 
 
@@ -111,7 +105,6 @@ public class UploadController implements UploadResource {
         accessControlService.verifyUserIsReviewerOrApprover(dossierId, fileId);
 
         try {
-
             reanalysisService.importRedactions(ByteContentDocument.builder().dossierId(dossierId).fileId(fileId).document(file.getBytes()).pages(pageInclusionRequest).build());
 
             auditPersistenceService.audit(AuditRequest.builder()
@@ -122,84 +115,116 @@ public class UploadController implements UploadResource {
                                                   .details(Map.of("dossierId", dossierId))
                                                   .build());
         } catch (IOException e) {
-            throw new BadRequestException(e.getMessage(), e);
+            throw new BadRequestException("Failed to import redactions: " + e.getMessage(), e);
         } catch (FeignException e) {
             throw processFeignException(e);
         }
     }
 
 
-    private String getExtension(String fileName) {
+    private void validateExtensionOrThrow(String extension) {
 
-        return fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase(Locale.ROOT);
+        if (!fileFormatValidationService.getAllFileFormats().contains(extension)) {
+            throw new BadRequestException("Invalid file uploaded (unrecognized extension).");
+        }
+        if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
+            throw new NotAllowedException("Insufficient permissions for this file type.");
+        }
     }
 
 
+    /**
+     * 1. Write the uploaded content to a temp ZIP file
+     * 2. Check the number of entries and reject if too big or if symlinks found
+     * 3. Unzip and process each file, while checking size and ratio.
+     */
     private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
 
-        File tempFile = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip");
-        try (var fileOutputStream = new FileOutputStream(tempFile)) {
-            IOUtils.write(fileContent, fileOutputStream);
+        File tempZip = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip");
+        try (FileOutputStream fos = new FileOutputStream(tempZip)) {
+            IOUtils.write(fileContent, fos);
         }
 
-        try {
-            checkForSymlinks(tempFile);
+        validateZipEntries(tempZip);
 
-            var zipData = unzip(tempFile, dossierId, keepManualRedactions, disableAutomaticAnalysis);
+        try {
+            ZipData zipData = processZipContents(tempZip, dossierId, keepManualRedactions, disableAutomaticAnalysis);
 
             if (zipData.csvBytes != null) {
                 try {
-                    var importResult = uploadService.importCsv(dossierId, zipData.csvBytes);
-                    zipData.fileUploadResult.getProcessedAttributes().addAll(importResult.getProcessedAttributes());
-                    zipData.fileUploadResult.getProcessedFileIds().addAll(importResult.getProcessedFileIds());
+                    FileUploadResult csvResult = uploadService.importCsv(dossierId, zipData.csvBytes);
+                    zipData.fileUploadResult.getProcessedAttributes().addAll(csvResult.getProcessedAttributes());
+                    zipData.fileUploadResult.getProcessedFileIds().addAll(csvResult.getProcessedFileIds());
                 } catch (Exception e) {
-                    log.debug("CSV file inside ZIP failed", e);
-                    // TODO return un-processed files to client
+                    log.debug("CSV file inside ZIP failed to import", e);
                 }
             } else if (zipData.fileUploadResult.getFileIds().isEmpty()) {
                 if (zipData.containedUnpermittedFiles) {
-                    throw new NotAllowedException("Zip file contains unpermitted files");
+                    throw new NotAllowedException("Zip file contains unpermitted files.");
                 } else {
-                    throw new BadRequestException("Only unsupported files in zip file");
+                    throw new BadRequestException("Only unsupported files in the ZIP.");
                 }
             }
 
             return zipData.fileUploadResult;
-
         } finally {
-            boolean isDeleted = tempFile.delete();
-            if (!isDeleted) {
-                log.warn("tempFile could not be deleted");
+
+            if (!tempZip.delete()) {
+                log.warn("Could not delete temporary ZIP file: {}", tempZip);
             }
         }
     }
 
 
-    private void checkForSymlinks(File tempFile) throws IOException {
+    private void validateZipEntries(File tempZip) throws IOException {
+
+        try (FileInputStream fis = new FileInputStream(tempZip); ZipFile zipFile = new ZipFile(fis.getChannel())) {
+
+            int count = 0;
+            var entries = zipFile.getEntries();
+            while (entries.hasMoreElements()) {
+                ZipArchiveEntry ze = entries.nextElement();
 
-        try (var fis = new FileInputStream(tempFile); var zipFile = new ZipFile(fis.getChannel())) {
-            for (var entryEnum = zipFile.getEntries(); entryEnum.hasMoreElements(); ) {
-                var ze = entryEnum.nextElement();
                 if (ze.isUnixSymlink()) {
-                    throw new BadRequestException("ZIP-files with symlinks are not allowed");
+                    throw new BadRequestException("ZIP-files with symlinks are not allowed.");
+                }
+
+                if (!ze.isDirectory() && !ze.getName().startsWith(".")) {
+                    count++;
+                    if (count > THRESHOLD_ENTRIES) {
+                        throw new BadRequestException("ZIP-Bomb detected: too many entries.");
+                    }
                 }
             }
         }
     }
 
 
-    private ZipData unzip(File tempFile, String dossierId, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
+    private ZipData processZipContents(File tempZip, String dossierId, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
 
-        var zipData = new ZipData();
+        ZipData zipData = new ZipData();
 
-        try (var fis = new FileInputStream(tempFile); var zipFile = new ZipFile(fis.getChannel())) {
+        try (FileInputStream fis = new FileInputStream(tempZip); ZipFile zipFile = new ZipFile(fis.getChannel())) {
 
-            for (var entryEnum = zipFile.getEntries(); entryEnum.hasMoreElements(); ) {
-                var ze = entryEnum.nextElement();
-                zipData.totalEntryArchive++;
+            var entries = zipFile.getEntries();
+            while (entries.hasMoreElements()) {
+                ZipArchiveEntry entry = entries.nextElement();
 
-                if (!ze.isDirectory()) {
-                    processFileZipEntry(ze, zipFile, dossierId, keepManualRedactions, zipData, disableAutomaticAnalysis);
+                if (entry.isDirectory() || entry.getName().startsWith(".")) {
+                    continue;
+                }
+
+                byte[] entryBytes = readEntryWithRatioCheck(entry, zipFile);
+                zipData.totalSizeArchive += entryBytes.length;
+                if (zipData.totalSizeArchive > THRESHOLD_SIZE) {
+                    throw new BadRequestException("ZIP-Bomb detected (exceeds total size limit).");
+                }
+
+                String extension = getExtension(entry.getName());
+                if ("csv".equalsIgnoreCase(extension)) {
+                    zipData.csvBytes = entryBytes;
+                } else {
+                    handleRegularFile(dossierId, entryBytes, extension, extractFileName(entry.getName()), zipData, keepManualRedactions, disableAutomaticAnalysis);
                 }
             }
         }
@@ -207,73 +232,70 @@ public class UploadController implements UploadResource {
     }
 
 
-    private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData, boolean disableAutomaticAnalysis) throws IOException {
+    private byte[] readEntryWithRatioCheck(ZipArchiveEntry entry, ZipFile zipFile) throws IOException {
 
-        var extension = getExtension(ze.getName());
+        long compressedSize = entry.getCompressedSize() > 0 ? entry.getCompressedSize() : 1;
+        try (var is = zipFile.getInputStream(entry); var bos = new ByteArrayOutputStream()) {
 
-        final String fileName;
-        if (ze.getName().lastIndexOf("/") >= 0) {
-            fileName = ze.getName().substring(ze.getName().lastIndexOf("/") + 1);
-        } else {
-            fileName = ze.getName();
-        }
+            byte[] buffer = new byte[4096];
+            int bytesRead;
+            int totalUncompressed = 0;
 
-        if (fileName.startsWith(".")) {
-            return;
-        }
+            while ((bytesRead = is.read(buffer)) != -1) {
+                bos.write(buffer, 0, bytesRead);
+                totalUncompressed += bytesRead;
 
-        var entryAsBytes = readCurrentZipEntry(ze, zipFile);
-        zipData.totalSizeArchive += entryAsBytes.length;
-
-        // 1. the uncompressed data size is too much for the application resource capacity
-        // 2. too many entries in the archive can lead to inode exhaustion of the file-system
-        if (zipData.totalSizeArchive > THRESHOLD_SIZE || zipData.totalEntryArchive > THRESHOLD_ENTRIES) {
-            throw new BadRequestException("ZIP-Bomb detected.");
-        }
-
-        if ("csv".equals(extension)) {
-            zipData.csvBytes = entryAsBytes;
-        } else if (fileFormatValidationService.getAllFileFormats().contains(extension)) {
-
-            if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
-                zipData.containedUnpermittedFiles = true;
-                return;
-            }
-            zipData.containedUnpermittedFiles = false;
-
-            try {
-                var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions, disableAutomaticAnalysis);
-                zipData.fileUploadResult.getFileIds().addAll(result.getFileIds());
-            } catch (Exception e) {
-                log.debug("PDF File inside ZIP failed", e);
-                // TODO return un-processed files to client
+                double ratio = (double) totalUncompressed / compressedSize;
+                if (ratio > THRESHOLD_RATIO) {
+                    throw new BadRequestException("ZIP-Bomb detected (compression ratio too high).");
+                }
             }
+            return bos.toByteArray();
         }
     }
 
 
-    private byte[] readCurrentZipEntry(ZipArchiveEntry ze, ZipFile zipFile) throws IOException {
+    private void handleRegularFile(String dossierId,
+                                   byte[] fileBytes,
+                                   String extension,
+                                   String fileName,
+                                   ZipData zipData,
+                                   boolean keepManualRedactions,
+                                   boolean disableAutomaticAnalysis) {
 
-        var bos = new ByteArrayOutputStream();
-
-        try (var entryStream = zipFile.getInputStream(ze)) {
-            var buffer = new byte[2048];
-            var nBytes = 0;
-            int totalSizeEntry = 0;
-
-            while ((nBytes = entryStream.read(buffer)) > 0) {
-                bos.write(buffer, 0, nBytes);
-                totalSizeEntry += nBytes;
-
-                double compressionRatio = (float) totalSizeEntry / ze.getCompressedSize();
-                if (compressionRatio > THRESHOLD_RATIO) {
-                    // ratio between compressed and uncompressed data is highly suspicious, looks like a Zip Bomb Attack
-                    throw new BadRequestException("ZIP-Bomb detected.");
-                }
-            }
+        if (!fileFormatValidationService.getAllFileFormats().contains(extension)) {
+            zipData.containedUnpermittedFiles = false;
+            return;
         }
 
-        return bos.toByteArray();
+        if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
+            zipData.containedUnpermittedFiles = true;
+            return;
+        }
+
+        try {
+            FileUploadResult result = uploadService.processSingleFile(dossierId, fileName, fileBytes, keepManualRedactions, disableAutomaticAnalysis);
+            zipData.fileUploadResult.getFileIds().addAll(result.getFileIds());
+        } catch (Exception e) {
+            log.debug("Failed to process file '{}' in ZIP: {}", fileName, e.getMessage(), e);
+        }
+    }
+
+
+    private String extractFileName(String path) {
+
+        int idx = path.lastIndexOf('/');
+        return (idx >= 0) ? path.substring(idx + 1) : path;
+    }
+
+
+    private String getExtension(String fileName) {
+
+        int idx = fileName.lastIndexOf('.');
+        if (idx < 0) {
+            return "";
+        }
+        return fileName.substring(idx + 1).toLowerCase(Locale.ROOT);
     }
 
 
@@ -282,7 +304,6 @@ public class UploadController implements UploadResource {
 
         byte[] csvBytes;
         int totalSizeArchive;
-        int totalEntryArchive;
         FileUploadResult fileUploadResult = new FileUploadResult();
         boolean containedUnpermittedFiles;
 

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UserStatsController.java

@@ -0,0 +1,60 @@
+package com.iqser.red.persistence.service.v1.external.api.impl.controller;
+
+import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_USER_STATS;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
+import com.iqser.red.service.persistence.service.v1.api.external.resource.UserStatsResource;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.UserStats;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RestController
+@RequiredArgsConstructor
+public class UserStatsController implements UserStatsResource {
+
+    private final UserService userService;
+    private final DossierService dossierService;
+    private final FileStatusPersistenceService fileStatusPersistenceService;
+    private final DossierACLService dossierACLService;
+
+
+    @Override
+    @PreAuthorize("hasAuthority('" + READ_USER_STATS + "')")
+    public ResponseEntity<UserStats> getUserStats(String userId) {
+
+        if (userService.getUserById(userId).isEmpty()) {
+            throw new NotFoundException(String.format("The user with id %s is not found.", userId));
+        }
+        List<String> dossierMemberships = new ArrayList<>();
+        List<String> dossierOwnerships = new ArrayList<>();
+        dossierService.getAllDossiers()
+                .stream()
+                .filter(dossierEntity -> dossierEntity.getHardDeletedTime() == null)
+                .forEach(d -> {
+                    if (dossierACLService.getMembers(d.getId()).contains(userId)) {
+                        dossierMemberships.add(d.getId());
+                    }
+                    if (dossierACLService.getOwners(d.getId()).contains(userId)) {
+                        dossierOwnerships.add(d.getId());
+                    }
+                });
+
+        return new ResponseEntity<>(new UserStats(dossierMemberships.size(), dossierOwnerships.size(), this.fileStatusPersistenceService.getNumberOfAssignedFiles(userId)),
+                                    HttpStatus.OK);
+    }
+
+}

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java

@@ -7,20 +7,26 @@ import static com.iqser.red.service.persistence.service.v2.api.external.resource
 
 import java.util.ArrayList;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.security.access.AccessDeniedException;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierController;
 import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController;
 import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMapper;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
 import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
+import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
@@ -28,13 +34,16 @@ import com.iqser.red.service.persistence.management.v1.processor.service.users.m
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.iqser.red.service.persistence.service.v2.api.external.model.BulkComponentsRequest;
 import com.iqser.red.service.persistence.service.v2.api.external.model.Component;
 import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentOverrideList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponentsList;
 import com.iqser.red.service.persistence.service.v2.api.external.resource.ComponentResource;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
+import com.knecon.fforesight.tenantcommons.TenantProvider;
 
+import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -45,15 +54,19 @@ import lombok.experimental.FieldDefaults;
 @Tag(name = "4. Component endpoints", description = "Provides operations related to components")
 public class ComponentControllerV2 implements ComponentResource {
 
+    private final AccessControlService accessControlService;
     private final ComponentLogService componentLogService;
     private final UserService userService;
     private final StatusController statusController;
     private final FileStatusService fileStatusService;
+    private final DossierController dossierController;
     private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
+    private final CurrentApplicationTypeProvider currentApplicationTypeProvider;
     private final ComponentMapper componentMapper = ComponentMapper.INSTANCE;
 
-    @Value("${application.type}")
-    private String applicationType;
+
+    @Value("${documine.components.filesLimit:100}")
+    private int documineComponentsFilesLimit = 100;
 
 
     @Override
@@ -92,12 +105,37 @@ public class ComponentControllerV2 implements ComponentResource {
         checkApplicationType();
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
         var dossierFiles = statusController.getDossierStatus(dossierId);
+
+        if(dossierFiles.size() > documineComponentsFilesLimit) {
+            throw new BadRequestException(String.format("The dossier you requested components for contains %s files this is above the limit of %s files for this endpoint, please use the POST %s", dossierFiles.size(), documineComponentsFilesLimit, FILE_PATH + BULK_COMPONENTS_PATH));
+        }
+
         return new FileComponentsList(dossierFiles.stream()
                                               .map(file -> getComponents(dossierTemplateId, dossierId, file.getFileId(), includeDetails))
                                               .toList());
     }
 
 
+    @Override
+    public FileComponentsList getComponentsForFiles(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
+                                             @PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                                             @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails,
+                                             @RequestBody BulkComponentsRequest bulkComponentsRequest){
+
+        if(bulkComponentsRequest.getFileIds().size() > documineComponentsFilesLimit) {
+            throw new BadRequestException(String.format("You requested components for %s files this is above the limit of %s files for this endpoint, lower the fileIds in the request", bulkComponentsRequest.getFileIds().size(), documineComponentsFilesLimit));
+        }
+
+        checkApplicationType();
+        dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
+        dossierController.getDossier(dossierId, false, false);
+        return new FileComponentsList(bulkComponentsRequest.getFileIds().stream()
+                                              .map(fileId -> getComponents(dossierTemplateId, dossierId, fileId, includeDetails))
+                                              .toList());
+
+    }
+
+
     @Override
     @PreAuthorize("hasAuthority('" + GET_RSS + "')")
     public void addOverride(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@@ -106,6 +144,7 @@ public class ComponentControllerV2 implements ComponentResource {
                             @RequestBody Component override) {
 
         checkApplicationType();
+        accessControlService.verifyUserIsReviewer(dossierId, fileId);
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
         componentLogService.overrideComponent(dossierId, fileId, componentMapper.toComponentLogEntry(override));
@@ -146,6 +185,7 @@ public class ComponentControllerV2 implements ComponentResource {
                                 @RequestBody RevertOverrideRequest revertOverrideRequest) {
 
         checkApplicationType();
+        accessControlService.verifyUserIsReviewer(dossierId, fileId);
         dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
 
         componentLogService.revertOverrides(dossierId, fileId, revertOverrideRequest);
@@ -154,7 +194,7 @@ public class ComponentControllerV2 implements ComponentResource {
 
     private void checkApplicationType() {
 
-        if(!applicationType.equals("DocuMine")) {
+        if (!currentApplicationTypeProvider.isDocuMine()) {
             throw new NotAllowedException("Components can only be accessed in DocuMine");
         }
     }

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierControllerV2.java

@@ -21,6 +21,7 @@ import com.iqser.red.persistence.service.v1.external.api.impl.controller.Downloa
 import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierAttributesManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
@@ -42,24 +43,25 @@ import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 
 @RestController
 @RequiredArgsConstructor
 @Tag(name = "2. Dossier endpoints", description = "Provides operations related to dossiers")
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DossierControllerV2 implements DossierResource {
 
-    private final DossierTemplateController dossierTemplateController;
-    private final DossierController dossierController;
-    private final AccessControlService accessControlService;
-    private final DossierAttributesManagementService dossierAttributesManagementService;
-    private final AuditPersistenceService auditPersistenceService;
-    private final DownloadController downloadController;
-    private final StatusController statusController;
-    private final DownloadStatusPersistenceService downloadStatusPersistenceService;
-
-    @Value("${application.type}")
-    private String applicationType;
+    DossierTemplateController dossierTemplateController;
+    DossierController dossierController;
+    AccessControlService accessControlService;
+    DossierAttributesManagementService dossierAttributesManagementService;
+    AuditPersistenceService auditPersistenceService;
+    DownloadController downloadController;
+    StatusController statusController;
+    DownloadStatusPersistenceService downloadStatusPersistenceService;
+    CurrentApplicationTypeProvider currentApplicationTypeProvider;
 
 
     public DossierList getDossiers(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@@ -191,7 +193,7 @@ public class DossierControllerV2 implements DossierResource {
                 .description(dossier.getDescription())
                 .ownerId(dossier.getOwnerId())
                 .memberIds(dossier.getMemberIds())
-                .approverIds(applicationType.equals("DocuMine") ? dossier.getMemberIds() : dossier.getApproverIds()) // for DocuMine, the members are always set as approvers
+                .approverIds(currentApplicationTypeProvider.isDocuMine() ? dossier.getMemberIds() : dossier.getApproverIds()) // for DocuMine, the members are always set as approvers
                 .downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
                 .reportTemplateIds(dossier.getReportTemplateIds())
                 .watermarkId(null)

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java

@@ -247,6 +247,7 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
                                                  .filterable(fileAttributeConfig.isFilterable())
                                                  .displayedInFileList(fileAttributeConfig.isDisplayedInFileList())
                                                  .build())
+                        .includeInCsvExport(fileAttributeConfig.isIncludeInCsvExport())
                         .build())
                 .toList();
 
@@ -449,11 +450,15 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
         return new DossierAttributeDefinitionList(dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId)
                                                           .stream()
                                                           .map(config -> DossierAttributeDefinition.builder()
-
                                                                   .id(config.getId())
                                                                   .name(config.getLabel())
                                                                   .type(config.getType())
                                                                   .reportingPlaceholder(config.getPlaceholder())
+                                                                  .displaySettings(DossierAttributeDefinition.DossierDisplaySettings.builder()
+                                                                                           .editable(config.isEditable())
+                                                                                           .filterable(config.isFilterable())
+                                                                                           .displayedInDossierList(config.isDisplayedInDossierList())
+                                                                                           .build())
                                                                   .build())
                                                           .toList());
     }

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/FileControllerV2.java

@@ -156,8 +156,6 @@ public class FileControllerV2 implements FileResource {
                                               @PathVariable(FILE_ID_PARAM) String fileId,
                                               @RequestBody DownloadRequest downloadRequest) {
 
-        fileStatusManagementService.getFileStatus(fileId, false);
-
         return prepareBulkDownload(dossierTemplateId,
                                    dossierId,
                                    BulkDownloadRequest.builder()

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java

@@ -223,10 +223,9 @@ public interface DictionaryResource {
     void changeFlags(@PathVariable(TYPE_PARAMETER_NAME) String type,
                      @PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
                      @PathVariable(DOSSIER_ID_PARAMETER_NAME) String dossierId,
-                     @RequestParam(value = "addToDictionary") boolean addToDictionary);
+                     @RequestParam(value = "addToDictionaryAction") boolean addToDictionaryAction);
 
 
-    @ResponseStatus(HttpStatus.ACCEPTED)
     @PostMapping(value = DICTIONARY_REST_PATH + DIFFERENCE + DOSSIER_TEMPLATE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Returns the difference between the dictionaries of the dossier template and all the dossiers inside the template for a list of given types.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Successfully returned DictionaryDifferenceResponse."), @ApiResponse(responseCode = "400", description = "The request is not valid.")})

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DossierResource.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.external.resource;
 
 import java.time.OffsetDateTime;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.springframework.http.HttpStatus;
@@ -17,6 +18,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierInformation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
@@ -29,10 +31,12 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
 public interface DossierResource {
 
     String DOSSIER_REST_PATH = ExternalApi.BASE_PATH + "/dossier";
+    String BY_ID_PATH = "/by-id";
     String DOSSIER_TEMPLATE_PATH = "/dossier-template";
     String DOSSIER_INFO_PATH = "/info";
     String DELETED_DOSSIERS_PATH = ExternalApi.BASE_PATH + "/deleted-dossiers";
     String CHANGES_DETAILS_PATH = "/changes/details";
+    String CHANGES_DETAILS_V2_PATH = "/changes/details/v2";
     String HARD_DELETE_PATH = "/hard-delete";
     String UNDELETE_PATH = "/restore";
 
@@ -62,6 +66,12 @@ public interface DossierResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success")})
     List<DossierChangeEntry> changesSince(@RequestBody JSONPrimitive<OffsetDateTime> since);
 
+    @ResponseBody
+    @PostMapping(value = DOSSIER_REST_PATH + CHANGES_DETAILS_V2_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "See if there are changes to dossiers since param", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success")})
+    DossierChangeResponseV2 changesSinceV2(@RequestBody JSONPrimitive<OffsetDateTime> since);
+
 
     @ResponseBody
     @PostMapping(value = DOSSIER_REST_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@@ -95,6 +105,13 @@ public interface DossierResource {
     List<Dossier> getDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
                               @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted);
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = DOSSIER_REST_PATH+BY_ID_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets dossiers by ids.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    JSONPrimitive<Map<String,Dossier>> getDossiersByIds(@RequestBody JSONPrimitive<Set<String>> dossierIds);
+
 
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/FileAttributesResource.java

@@ -44,7 +44,11 @@ public interface FileAttributesResource {
     String FILE_ID = "fileId";
     String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
 
-    Set<String> encodingList = Sets.newHashSet("ISO", "ASCII", "UTF-8");
+    String UTF_ENCODING = "UTF-8";
+    String ASCII_ENCODING = "ASCII";
+    String ISO_ENCODING = "ISO-8859-1";
+
+    Set<String> encodingList = Sets.newHashSet(ISO_ENCODING, ASCII_ENCODING, UTF_ENCODING);
 
 
     @ResponseBody

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/MigrationStatusResource.java

@@ -1,56 +0,0 @@
-package com.iqser.red.service.persistence.service.v1.api.external.resource;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.saas.migration.MigrationStatusResponse;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
-
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
-
-public interface MigrationStatusResource {
-
-    String MIGRATION_STATUS_REST_PATH = ExternalApi.BASE_PATH + "/migration-status";
-    String START_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/start_migration";
-    String REVERT_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/revert_migration";
-    String RETRY_MIGRATION_REST_PATH = ExternalApi.BASE_PATH + "/retry_migration";
-
-    String FILE_ID = "fileId";
-    String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
-
-    String DOSSIER_ID = "dossierId";
-    String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
-
-
-    @ResponseBody
-    @PostMapping(value = MIGRATION_STATUS_REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Show the status of the migration", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    MigrationStatusResponse migrationStatus();
-
-
-    @ResponseBody
-    @PostMapping(value = START_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
-    @Operation(summary = "Start SAAS migration for specific file", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> startMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
-
-
-    @ResponseBody
-    @PostMapping(value = REVERT_MIGRATION_REST_PATH + FILE_ID_PATH_VARIABLE + DOSSIER_ID_PATH_VARIABLE)
-    @Operation(summary = "Start SAAS migration for specific file", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> revertMigrationForFile(@RequestParam(value = DOSSIER_ID) String dossierId, @RequestParam(value = FILE_ID) String fileId);
-
-
-    @ResponseBody
-    @PostMapping(value = RETRY_MIGRATION_REST_PATH)
-    @Operation(summary = "Restart SAAS migration for all files in error state", description = "None")
-    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Success.")})
-    ResponseEntity<?> requeueErrorFiles();
-
-}

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ReanalysisResource.java

@@ -38,6 +38,7 @@ public interface ReanalysisResource {
 
     String EXCLUDED_STATUS_PARAM = "excluded";
     String FORCE_PARAM = "force";
+    String ALL_PAGES = "allPages";
 
 
     @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE)
@@ -73,7 +74,8 @@ public interface ReanalysisResource {
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "409", description = "Conflict"), @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "403", description = "Forbidden"), @ApiResponse(responseCode = "400", description = "Cannot OCR approved file")})
     void ocrFile(@PathVariable(DOSSIER_ID) String dossierId,
                  @PathVariable(FILE_ID) String fileId,
-                 @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force);
+                 @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force,
+                 @RequestParam(value = ALL_PAGES, required = false, defaultValue = FALSE) boolean allPages);
 
 
     @Operation(summary = "Ocr and reanalyze multiple files for a dossier", description = "None")

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/RulesResource.java

@@ -6,6 +6,7 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RequestPart;
@@ -28,6 +29,7 @@ public interface RulesResource {
     String RULES_PATH = ExternalApi.BASE_PATH + "/rules";
     String UPLOAD_PATH = "/upload";
     String DOWNLOAD_PATH = "/download";
+    String RESET_PATH = "/reset";
 
     String DOSSIER_TEMPLATE_PARAMETER_NAME = "dossierTemplateId";
     String DOSSIER_TEMPLATE_PATH_VARIABLE = "/{dossierTemplateId}";
@@ -105,4 +107,10 @@ public interface RulesResource {
     @GetMapping(value = RULES_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE + RULE_FILE_TYPE_PATH_VARIABLE + DOWNLOAD_PATH)
     ResponseEntity<?> downloadFile(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType);
 
+    @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
+    @Operation(summary = "Resets the timeout detected flag in a Rule file.")
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "No content")})
+    @PutMapping(value = RULES_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE + RULE_FILE_TYPE_PATH_VARIABLE + RESET_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    void unlockRules(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @PathVariable(RULE_FILE_TYPE_PARAMETER_NAME) RuleFileType ruleFileType);
 }

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/StatusResource.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.external.resource;
 import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -16,6 +17,7 @@ import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.warning.ApproveResponse;
 
 import io.swagger.v3.oas.annotations.Operation;
@@ -25,6 +27,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
 public interface StatusResource {
 
     String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status";
+    String BY_ID_PATH = "/by-id";
     String CHANGES_SINCE_PATH = "/changes";
     String BULK_REST_PATH = "/bulk";
     String ASSIGNEE_REST_PATH = "/set-assignee";
@@ -56,6 +59,14 @@ public interface StatusResource {
     Map<String, List<FileStatus>> getDossierStatus(@RequestBody List<String> dossierIds);
 
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @PostMapping(value = STATUS_REST_PATH + BY_ID_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets the status for files by dossierId and fileIds.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    JSONPrimitive<Map<String, List<FileStatus>>> getFilesByIds(@RequestBody JSONPrimitive<Map<String, Set<String>>> filesByDossier);
+
+
     @ResponseStatus(value = HttpStatus.OK)
     @ResponseBody
     @PostMapping(value = STATUS_REST_PATH + DELETED_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/UserStatsResource.java

@@ -0,0 +1,37 @@
+package com.iqser.red.service.persistence.service.v1.api.external.resource;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.UserStats;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+
+@ApiResponses(value = {@ApiResponse(responseCode = "429", description = "Too many requests.")})
+public interface UserStatsResource {
+
+    String USER_ID_PARAM = "userId";
+
+    String USER_ID_PATH_VARIABLE = "/{" + USER_ID_PARAM + "}";
+
+    String STATS_PATH = "/user-stats";
+
+    String PATH = ExternalApi.BASE_PATH + STATS_PATH;
+
+
+    @ResponseStatus(value = HttpStatus.OK)
+    @ResponseBody
+    @GetMapping(value = PATH + USER_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Gets user stats for user specified by id.", description = "")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found")})
+    ResponseEntity<UserStats> getUserStats(@Parameter(name = USER_ID_PARAM, description = "The unique identifier of the user whose statistics we want to retrieve.", required = true) @PathVariable(USER_ID_PARAM) String userId);
+
+}

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/BulkComponentsRequest.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.persistence.service.v2.api.external.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class BulkComponentsRequest {
+
+    private List<String> fileIds = new ArrayList<>();
+}

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/DossierAttributeDefinition.java

@@ -17,5 +17,18 @@ public class DossierAttributeDefinition {
     private String name;
     private DossierAttributeType type;
     private String reportingPlaceholder;
+    private DossierDisplaySettings displaySettings;
+
+    @Data
+    @NoArgsConstructor
+    @AllArgsConstructor
+    @Builder
+    public static class DossierDisplaySettings {
+
+        private boolean editable;
+        private boolean filterable;
+        private boolean displayedInDossierList;
+
+    }
 
 }

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/FileAttributeDefinition.java

@@ -20,6 +20,7 @@ public class FileAttributeDefinition {
     private String mappedCsvColumnHeader;
     private String reportingPlaceholder;
     private DisplaySettings displaySettings;
+    private boolean includeInCsvExport;
 
     @Data
     @NoArgsConstructor

persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java

@@ -20,6 +20,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
+import com.iqser.red.service.persistence.service.v2.api.external.model.BulkComponentsRequest;
 import com.iqser.red.service.persistence.service.v2.api.external.model.Component;
 import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentOverrideList;
 import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents;
@@ -74,6 +75,16 @@ public interface ComponentResource {
                                               @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails);
 
 
+    @PostMapping(value = FILE_PATH
+                         + BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}, consumes = MediaType.APPLICATION_JSON_VALUE)
+    @Operation(summary = "Returns the components for all files of a dossier", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
+    FileComponentsList getComponentsForFiles(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
+                                             @Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier that contains the file.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                                             @Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails,
+                                             @RequestBody BulkComponentsRequest bulkComponentsRequest);
+
+
     @ResponseBody
     @ResponseStatus(value = HttpStatus.NO_CONTENT)
     @PostMapping(value = FILE_PATH + FILE_ID_PATH_VARIABLE + OVERRIDES_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml

@@ -1626,6 +1626,50 @@ paths:
           $ref: '#/components/responses/429'
         "500":
           $ref: '#/components/responses/500'
+    post:
+      operationId: getComponentsForFiles
+      tags:
+        - 4. Components
+      summary: Returns the FileComponents for requested files
+      description: |
+        This endpoint fetches components for the requested files by its ids. Like individual file components, 
+        these represent various aspects, metadata or content of the files. Entity and component rules define these components based on the file's
+        content. They can give a *structured view* on a document's text.
+        
+         To include detailed component information, set the `includeDetails` query parameter to `true`.
+      parameters:
+        - $ref: '#/components/parameters/dossierTemplateId'
+        - $ref: '#/components/parameters/dossierId'
+        - $ref: '#/components/parameters/includeComponentDetails'
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/BulkComponentsRequest'
+        required: true
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/FileComponentsList'
+            application/xml:
+              schema:
+                $ref: '#/components/schemas/FileComponentsList'
+          description: |
+            Successfully fetched components for all files in the dossier.
+        "400":
+          $ref: '#/components/responses/400'
+        "401":
+          $ref: '#/components/responses/401'
+        "403":
+          $ref: '#/components/responses/403'
+        "404":
+          $ref: '#/components/responses/404-dossier'
+        "429":
+          $ref: '#/components/responses/429'
+        "500":
+          $ref: '#/components/responses/500'
   /api/downloads:
     get:
       operationId: getDownloadStatusList
@@ -2773,6 +2817,16 @@ components:
                         entityRuleId: DEF.13.37
                         type: another_entity_type
                         page: 456
+    BulkComponentsRequest:
+      type: object
+      description: Request payload to get components for multiple files.
+      properties:
+        fileIds:
+          type: array
+          description: A list with unique identifiers of the files for which components should be retrieved.
+          items:
+            type: string
+            description: The unique identifier of a file.
     DossierStatusDefinition:
       type: object
       description: |
@@ -2864,6 +2918,8 @@ components:
             placeholder follows a specific format convention:
             `{{dossier.attribute.<name>}}` while the name is transformed into 'PascalCase' and does not contain 
             whitespaces. The placeholder is unique in a dossier template.
+        displaySettings:
+          $ref: '#/components/schemas/DossierAttributeDisplaySettings'
       required:
         - name
         - type
@@ -2872,6 +2928,35 @@ components:
         name: "Document Summary"
         type: "TEXT"
         reportingPlaceholder: "{{dossier.attribute.DocumentSummary}}"
+        displaySettings:
+          editable: true
+          filterable: false
+          displayedInDossierList: false
+    DossierAttributeDisplaySettings:
+      type: object
+      description: |
+        Display setting for the user interface. These settings control how the UI handles and presents the dossier attributes.
+      properties:
+        editable:
+          type: boolean
+          description: |
+            If `true`, the user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible.
+        filterable:
+          type: boolean
+          description: |
+            If `true`, the user interfaces add filter options to the dossier list.
+        displayedInDossierList:
+          type: boolean
+          description: |
+            if `true`, the user interfaces show the values in the dossier list.
+      required:
+        - editable
+        - filterable
+        - displayedInDossierList
+      example:
+        editable: true
+        filterable: true
+        displayedInDossierList: false
     FileAttributeDefinition:
       type: object
       description: |
@@ -2994,10 +3079,18 @@ components:
             name: "Dossier Summary"
             type: "TEXT"
             reportingPlaceholder: "{{dossier.attribute.DossierSummary}}"
+            displaySettings:
+              editable: true
+              filterable: false
+              displayedInFileList: false
           - id: "23e45678-e90b-12d3-a456-765114174321"
             name: "Comment"
             type: "TEXT"
             reportingPlaceholder: "{{dossier.attribute.Comment}}"
+            displaySettings:
+              editable: true
+              filterable: false
+              displayedInFileList: false
     FileAttributeDefinitionList:
       type: object
       description: A list of file attribute definitions.

persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml

@@ -1492,6 +1492,8 @@ components:
             placeholder follows a specific format convention:
             `{{dossier.attribute.<name>}}` while the name is transformed into 'PascalCase' and does not contain 
             whitespaces. The placeholder is unique in a dossier template.
+        displaySettings:
+          $ref: '#/components/schemas/DossierAttributeDisplaySettings'
       required:
         - name
         - type
@@ -1500,6 +1502,35 @@ components:
         name: "Document Summary"
         type: "TEXT"
         reportingPlaceholder: "{{dossier.attribute.DocumentSummary}}"
+        displaySettings:
+          editable: true
+          filterable: false
+          displayedInDossierList: false
+    DossierAttributeDisplaySettings:
+      type: object
+      description: |
+        Display setting for the user interface. These settings control how the UI handles and presents the dossier attributes.
+      properties:
+        editable:
+          type: boolean
+          description: |
+            If `true`, the user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible.
+        filterable:
+          type: boolean
+          description: |
+            If `true`, the user interfaces add filter options to the dossier list.
+        displayedInDossierList:
+          type: boolean
+          description: |
+            if `true`, the user interfaces show the values in the dossier list.
+      required:
+        - editable
+        - filterable
+        - displayedInDossierList
+      example:
+        editable: true
+        filterable: true
+        displayedInDossierList: false
     FileAttributeDefinition:
       type: object
       description: |
@@ -1622,10 +1653,18 @@ components:
             name: "Dossier Summary"
             type: "TEXT"
             reportingPlaceholder: "{{dossier.attribute.DossierSummary}}"
+            displaySettings:
+              editable: true
+              filterable: false
+              displayedInFileList: false
           - id: "23e45678-e90b-12d3-a456-765114174321"
             name: "Comment"
             type: "TEXT"
             reportingPlaceholder: "{{dossier.attribute.Comment}}"
+            displaySettings:
+              editable: true
+              filterable: false
+              displayedInFileList: false
     FileAttributeDefinitionList:
       type: object
       description: A list of file attribute definitions.

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierAttributesConfigInternalController.java

@@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.DossierAttributeConfigMapper;
 import com.iqser.red.service.persistence.service.v1.api.internal.resources.DossierAttributesConfigResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
 
@@ -27,7 +28,7 @@ public class DossierAttributesConfigInternalController implements DossierAttribu
     @Override
     public List<DossierAttributeConfig> getDossierAttributes(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
 
-        return convert(dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId), DossierAttributeConfig.class);
+        return convert(dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId), DossierAttributeConfig.class, new DossierAttributeConfigMapper());
     }
 
 }

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileAttributesConfigInternalController.java

@@ -8,6 +8,7 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileAttributeConfigMapper;
 import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileAttributesConfigResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
 
@@ -25,7 +26,7 @@ public class FileAttributesConfigInternalController implements FileAttributesCon
     @Override
     public List<FileAttributeConfig> getFileAttributeConfigs(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
 
-        return convert(fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId), FileAttributeConfig.class);
+        return convert(fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId), FileAttributeConfig.class, new FileAttributeConfigMapper());
     }
 
 }

persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java

@@ -57,7 +57,7 @@ public class AdminInterfaceController {
 
         fileStatusService.validateFileIsNotDeletedAndNotApproved(fileId);
 
-        fileStatusService.setStatusOcrQueued(dossierId, fileId);
+        fileStatusService.setStatusOcrQueued(dossierId, fileId, false);
 
     }
 
@@ -91,7 +91,7 @@ public class AdminInterfaceController {
 
                     if (!dryRun) {
                         fileStatusService.validateFileIsNotDeletedAndNotApproved(file.getId());
-                        fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId());
+                        fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId(), false);
                     }
 
                 }

persistence-service-v1/persistence-service-processor-v1/build.gradle.kts

@@ -14,6 +14,7 @@ configurations {
     }
 }
 dependencies {
+
     api(project(":persistence-service-shared-api-v1"))
     api(project(":persistence-service-shared-mongo-v1"))
     api(project(":persistence-service-external-api-v1"))
@@ -30,22 +31,18 @@ dependencies {
         exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
         exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
     }
-    api("com.knecon.fforesight:layoutparser-service-internal-api:0.181.0") {
-        exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
-        exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
-    }
     api("com.iqser.red.service:search-service-api-v1:${rootProject.extra.get("searchServiceVersion")}") {
         exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
         exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
     }
     api("com.knecon.fforesight:azure-ocr-service-api:0.13.0")
-    implementation("com.knecon.fforesight:llm-service-api:1.17.0")
-    api("com.knecon.fforesight:jobs-commons:0.10.0")
+    implementation("com.knecon.fforesight:llm-service-api:1.20.0-RED10072.2")
+    api("com.knecon.fforesight:jobs-commons:0.13.0")
     api("com.iqser.red.commons:storage-commons:2.50.0")
-    api("com.knecon.fforesight:tenant-commons:0.31.0") {
+    api("com.knecon.fforesight:tenant-commons:0.31.0-RED10196.0") {
         exclude(group = "com.iqser.red.commons", module = "storage-commons")
     }
-    api("com.knecon.fforesight:database-tenant-commons:0.28.0") {
+    api("com.knecon.fforesight:database-tenant-commons:0.31.0") {
         exclude(group = "com.knecon.fforesight", module = "tenant-commons")
     }
     api("com.knecon.fforesight:keycloak-commons:0.30.0") {
@@ -74,7 +71,6 @@ dependencies {
     api("commons-validator:commons-validator:1.7")
     api("com.opencsv:opencsv:5.9")
 
-    implementation("com.google.protobuf:protobuf-java:4.27.1")
     implementation("org.mapstruct:mapstruct:1.6.2")
     annotationProcessor("org.mapstruct:mapstruct-processor:1.6.2")
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/RedObjectIdentityRetrievalStrategy.java

@@ -5,6 +5,7 @@ import org.springframework.security.acls.model.ObjectIdentity;
 import org.springframework.security.acls.model.ObjectIdentityRetrievalStrategy;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.IHavingDossierId;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 
 import lombok.extern.slf4j.Slf4j;
@@ -18,8 +19,8 @@ public class RedObjectIdentityRetrievalStrategy implements ObjectIdentityRetriev
         log.debug("Requesting data for object: {}", domainObject);
         if (domainObject instanceof Dossier) {
             return new ObjectIdentityImpl("Dossier", ((Dossier) domainObject).getId());
-        } else if (domainObject instanceof DossierChangeEntry) {
-            return new ObjectIdentityImpl("Dossier", ((DossierChangeEntry) domainObject).getDossierId());
+        } else if (domainObject instanceof IHavingDossierId) {
+            return new ObjectIdentityImpl("Dossier", ((IHavingDossierId) domainObject).getDossierId());
         } else if (domainObject instanceof String) {
             // TODO ACL this will not work once we have more than one type.
             return new ObjectIdentityImpl("Dossier", (String) domainObject);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/dossier/DossierACLService.java

@@ -73,6 +73,24 @@ public class DossierACLService extends AbstractACLService<String> {
     }
 
 
+    public Set<String> getOwners(String dossierId) {
+
+        ObjectIdentityImpl dossierIdentity = new ObjectIdentityImpl(getIdentifier(), dossierId);
+        var acl = mutableAclService.readAclById(dossierIdentity);
+        Set<String> members = new HashSet<>();
+        acl.getEntries()
+                .forEach(entry -> {
+                    if (entry.getSid() instanceof PrincipalSid) {
+                        var principal = ((PrincipalSid) entry.getSid()).getPrincipal();
+                        if (entry.getPermission().getMask() == RedPermission.OWNER.getMask()) {
+                            members.add(principal);
+                        }
+                    }
+                });
+        return members;
+    }
+
+
     @Override
     public String getIdentifier() {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java

@@ -53,6 +53,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.DossierAttributeConfigMapper;
+import com.iqser.red.service.persistence.management.v1.processor.utils.FileAttributeConfigMapper;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -183,13 +185,16 @@ public class DossierTemplateExportService {
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
                                                                                       getFilename(ExportFilename.DOSSIER_ATTRIBUTES_CONFIG, JSON_EXT),
                                                                                       objectMapper.writeValueAsBytes(convert(dossierAttributesConfig,
-                                                                                                                             DossierAttributeConfig.class))));
+                                                                                                                             DossierAttributeConfig.class,
+                                                                                                                             new DossierAttributeConfigMapper()))));
 
         // add file attribute configs
         List<FileAttributeConfigEntity> fileAttributeConfigList = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplate.getId());
         fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,
                                                                                       getFilename(ExportFilename.FILE_ATTRIBUTE_CONFIG, JSON_EXT),
-                                                                                      objectMapper.writeValueAsBytes(convert(fileAttributeConfigList, FileAttributeConfig.class))));
+                                                                                      objectMapper.writeValueAsBytes(convert(fileAttributeConfigList,
+                                                                                                                             FileAttributeConfig.class,
+                                                                                                                             new FileAttributeConfigMapper()))));
 
         // add legal basis mapping
         List<LegalBasisEntity> legalBasisMappingList = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplate.getId());
@@ -275,8 +280,7 @@ public class DossierTemplateExportService {
         // and 1 txt file for every type: entries, false positives and false recommendation
 
         // remove the types related to dossiers and also the ones that are deleted
-        // also the ones that are system - managed
-        List<TypeEntity> dossierTypes = dictionaryPersistenceService.getAllTypesForDossierTemplateWithoutSystemManaged(dossierTemplate.getId(), false);
+        List<TypeEntity> dossierTypes = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplate.getId(), false);
         for (TypeEntity typeEntity : dossierTypes) {
 //                log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId() + " " + typeEntity.isDeleted());
             entityTypeExportService.addEntityTypeToArchive(fileSystemBackedArchiver, typeEntity, folder);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateImportService.java

@@ -19,10 +19,14 @@ import java.util.stream.Collectors;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ComponentMappingImportModel;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.EntityTypeImportModel;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DossierTemplateArchiveReader;
+import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
@@ -33,14 +37,11 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ComponentMappingImportModel;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ImportTemplateResult;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DossierTemplateArchiveReader;
-import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.DateFormatsValidationService;
+import com.iqser.red.service.persistence.management.v1.processor.service.LayoutParsingTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.DefaultDateFormatsProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReportTemplateService;
 import com.iqser.red.service.persistence.management.v1.processor.service.RulesValidationService;
@@ -72,43 +73,44 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.redaction.v1.model.DroolsValidation;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DossierTemplateImportService {
 
-    @Value("${application.type}")
-    private String applicationType;
-
-    private final DossierTemplateRepository dossierTemplateRepository;
-    private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
-    private final RulesPersistenceService rulesPersistenceService;
-    private final DateFormatsPersistenceService dateFormatsPersistenceService;
-    private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
-    private final DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
-    private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
-    private final ColorsService colorsService;
-    private final DossierStatusPersistenceService dossierStatusPersistenceService;
-    private final WatermarkService watermarkService;
-    private final ReportTemplateService reportTemplateService;
-    private final ReportTemplatePersistenceService reportTemplatePersistenceService;
-    private final RulesValidationService rulesValidationService;
-    private final DateFormatsValidationService dateFormatsValidationService;
-    private final StorageService storageService;
-    private final FileManagementServiceSettings settings;
-    private final ComponentMappingService componentMappingService;
-    private final ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
-    private final EntityTypeImportService entityTypeImportService;
-    private final SystemManagedTypesImport systemManagedTypesImport;
-    private final DefaultDateFormatsProvider defaultDateFormatsProvider;
+    DossierTemplateRepository dossierTemplateRepository;
+    LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
+    RulesPersistenceService rulesPersistenceService;
+    DateFormatsPersistenceService dateFormatsPersistenceService;
+    DossierTemplatePersistenceService dossierTemplatePersistenceService;
+    DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
+    FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
+    ColorsService colorsService;
+    DossierStatusPersistenceService dossierStatusPersistenceService;
+    WatermarkService watermarkService;
+    ReportTemplateService reportTemplateService;
+    ReportTemplatePersistenceService reportTemplatePersistenceService;
+    RulesValidationService rulesValidationService;
+    DateFormatsValidationService dateFormatsValidationService;
+    StorageService storageService;
+    FileManagementServiceSettings settings;
+    ComponentMappingService componentMappingService;
+    ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
+    EntityTypeImportService entityTypeImportService;
+    SystemManagedTypesImport systemManagedTypesImport;
+    LayoutParsingTypeProvider layoutParsingTypeProvider;
+    CurrentApplicationTypeProvider currentApplicationTypeProvider;
+    DefaultDateFormatsProvider defaultDateFormatsProvider;
 
 
     public String importDossierTemplate(ImportDossierTemplateRequest request) {
@@ -147,6 +149,10 @@ public class DossierTemplateImportService {
         int importStep = 1;
         var dossierTemplateMeta = request.getDossierTemplate();
 
+        if (dossierTemplateMeta.getLayoutParsingType() == null) {
+            dossierTemplateMeta.setLayoutParsingType(layoutParsingTypeProvider.deferFromCurrentApplicationType());
+        }
+
         TemplateImportInfo templateImportInfo = TemplateImportInfo.builder().build();
 
         DossierTemplateEntity existingDossierTemplate = null;
@@ -285,7 +291,7 @@ public class DossierTemplateImportService {
                 importStep = 12;
                 entityTypeImportService.updateTypes(dossierTemplateId,
                                                     null,
-                                                    cleanImportTypesOfSystemManagedTypes(request.getEntityTypeImportModel(), systemManagedTypesImport.getSystemManagedTypeList()));
+                                                    entityTypeImportService.updateAndCleanSystemManagedFromImportTypes(request.getEntityTypeImportModel(), dossierTemplateId));
 
             } else {
 
@@ -302,7 +308,6 @@ public class DossierTemplateImportService {
                 dossierTemplateEntity.setId(UUID.randomUUID().toString());
                 dossierTemplateEntity.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
                 dossierTemplateEntity.setCreatedBy(request.getUserId());
-                dossierTemplateEntity.setLayoutParsingType(deferFromApplicationType());
 
                 var loadedDossierTemplate = dossierTemplateRepository.save(dossierTemplateEntity);
                 loadedDossierTemplate.setDossierTemplateStatus(dossierTemplatePersistenceService.computeDossierTemplateStatus(loadedDossierTemplate));
@@ -402,15 +407,9 @@ public class DossierTemplateImportService {
 
                 //set types
                 importStep = 12;
-                // import system managed entity types
-                EntityTypeImportModel systemManagedTypesModel = new EntityTypeImportModel();
-                systemManagedTypesModel.getTypes().addAll(systemManagedTypesImport.getSystemManagedTypeList());
-                entityTypeImportService.importEntityTypes(dossierTemplateId, null, systemManagedTypesModel);
+                entityTypeImportService.addMissingSystemManagedTypesToImport(request.getEntityTypeImportModel(), systemManagedTypesImport.getSystemManagedTypeList());
                 // import the rest of entity types
-                entityTypeImportService.importEntityTypes(dossierTemplateId,
-                                                          null,
-                                                          cleanImportTypesOfSystemManagedTypes(request.getEntityTypeImportModel(),
-                                                                                               systemManagedTypesImport.getSystemManagedTypeList()));
+                entityTypeImportService.importEntityTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
 
             }
 
@@ -496,7 +495,7 @@ public class DossierTemplateImportService {
     private void setDataFormats(ImportTemplateResult request, String dossierTemplateId) {
 
         String dateFormats = request.getDateFormats();
-        if (dateFormats == null && applicationType.equals("DocuMine")) {
+        if (dateFormats == null && currentApplicationTypeProvider.isDocuMine()) {
             dateFormats = defaultDateFormatsProvider.getDateFormats();
         }
         if (dateFormats != null) {
@@ -523,7 +522,7 @@ public class DossierTemplateImportService {
         dossierTemplateEntity.setModifiedBy(userId);
         dossierTemplateEntity.setValidFrom(dossierTemplate.getValidFrom() == null ? dossierTemplateEntity.getValidFrom() : dossierTemplate.getValidFrom());
         dossierTemplateEntity.setValidTo(dossierTemplate.getValidTo() == null ? dossierTemplateEntity.getValidTo() : dossierTemplate.getValidTo());
-        dossierTemplateEntity.setLayoutParsingType(deferFromApplicationType());
+        dossierTemplateEntity.setLayoutParsingType(layoutParsingTypeProvider.deferFromCurrentApplicationType());
 
         dossierTemplateEntity.setDownloadFileTypes(dossierTemplate.getDownloadFileTypes() == null || dossierTemplate.getDownloadFileTypes()
                 .isEmpty() ? dossierTemplateEntity.getDownloadFileTypes() : dossierTemplate.getDownloadFileTypes());
@@ -531,12 +530,6 @@ public class DossierTemplateImportService {
     }
 
 
-    private LayoutParsingType deferFromApplicationType() {
-
-        return Objects.equals(applicationType, "DocuMine") ? LayoutParsingType.DOCUMINE_OLD : LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH;
-    }
-
-
     private void setColors(String dossierTemplateId, Colors requestedColors) {
         // set colors
         if (requestedColors != null) {
@@ -680,30 +673,4 @@ public class DossierTemplateImportService {
         return dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
     }
 
-
-    private EntityTypeImportModel cleanImportTypesOfSystemManagedTypes(EntityTypeImportModel importModel, List<Type> systemManagedTypes) {
-
-        Set<String> systemManagedTypesByName = systemManagedTypes.stream()
-                .map(Type::getType)
-                .collect(Collectors.toSet());
-        Set<String> systemManagedTypesIdsFromImport = importModel.getTypes()
-                .stream()
-                .filter(t -> systemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
-                .map(Type::getType)
-                .collect(Collectors.toSet());
-
-        if (!systemManagedTypesIdsFromImport.isEmpty()) {
-
-            importModel.getTypes().removeIf(t -> systemManagedTypesIdsFromImport.contains(t.getType()));
-
-            importModel.getEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
-            importModel.getFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
-            importModel.getFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
-            importModel.getDeletedEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
-            importModel.getDeletedFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
-            importModel.getDeletedFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
-        }
-        return importModel;
-    }
-
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/EntityTypeImportService.java

@@ -125,10 +125,77 @@ public class EntityTypeImportService {
                             returnedType.getType(),
                             DictionaryEntryType.FALSE_RECOMMENDATION);
 
-            dictionaryPersistenceService.setVersion(returnedType.getTypeId(), type.getVersion());
+            dictionaryPersistenceService.setVersions(returnedType.getTypeId(), type.getVersion(), type.getAiCreationVersion());
         }
     }
 
+    public void addMissingSystemManagedTypesToImport(EntityTypeImportModel importModel, List<Type> systemManagedTypes) {
+        Set<String> defaultSystemManagedTypesByName = systemManagedTypes.stream()
+                .map(Type::getType)
+                .collect(Collectors.toSet());
+        importModel.getTypes()
+                .stream()
+                .filter(t -> defaultSystemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
+                .forEach(t -> {
+                    if (!t.isSystemManaged()) {
+                        t.setSystemManaged(true);
+                    }
+                });
+
+        Set<String> systemManagedTypesFromImport = importModel.getTypes()
+                .stream()
+                .filter(t -> defaultSystemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
+                .map(Type::getType)
+                .collect(Collectors.toSet());
+        List<Type> systemManagedTypesNotIncludedInTheImport = systemManagedTypes.stream()
+                .filter(type -> !systemManagedTypesFromImport.contains(type.getType()))
+                .collect(Collectors.toList());
+        importModel.getTypes().addAll(systemManagedTypesNotIncludedInTheImport);
+    }
+
+    public EntityTypeImportModel updateAndCleanSystemManagedFromImportTypes(EntityTypeImportModel importModel, String dossierTemplateId) {
+
+        List<TypeEntity> currentSystemManagedTypes = dossierTemplatePersistenceService.getTypesForDossierTemplate(dossierTemplateId)
+                .stream()
+                .filter(t -> t.getDossierId() == null)
+                .filter(t -> t.isSystemManaged())
+                .collect(Collectors.toList());
+
+        var currentSystemManagedTypesNameToTypeMap = currentSystemManagedTypes.stream()
+                .collect(Collectors.toMap(TypeEntity::getType, Function.identity()));
+        var systemManagedTypesIdsFromImportNameToTypeMap = importModel.getTypes()
+                .stream()
+                .filter(t -> currentSystemManagedTypesNameToTypeMap.keySet().contains(t.getType()) || t.isSystemManaged())
+                .collect(Collectors.toMap(Type::getType, Function.identity()));
+
+        if (!systemManagedTypesIdsFromImportNameToTypeMap.isEmpty()) {
+            systemManagedTypesIdsFromImportNameToTypeMap.entrySet().forEach(entry -> {
+                var typeId = currentSystemManagedTypesNameToTypeMap.get(entry.getKey()).getId();
+                entry.getValue().setDossierTemplateId(dossierTemplateId);
+                entry.getValue().setDossierId(null);
+                dictionaryManagementService.updateTypeValue(typeId, entry.getValue());
+
+                this.addEntries(importModel.getEntries(), importModel.getDeletedEntries(), typeId, entry.getKey(), DictionaryEntryType.ENTRY);
+                this.addEntries(importModel.getFalsePositives(), importModel.getDeletedFalsePositives(), typeId, entry.getKey(), DictionaryEntryType.FALSE_POSITIVE);
+                this.addEntries(importModel.getFalseRecommendations(), importModel.getDeletedFalseRecommendations(), typeId, entry.getKey(), DictionaryEntryType.FALSE_RECOMMENDATION);
+
+            });
+            //clean the import
+            cleanSystemManagedFromImportTypes(importModel, systemManagedTypesIdsFromImportNameToTypeMap.keySet());
+        }
+        return importModel;
+    }
+
+    private void cleanSystemManagedFromImportTypes(EntityTypeImportModel importModel, Set<String> systemManagedTypesIdsFromImport) {
+        importModel.getTypes().removeIf(t -> systemManagedTypesIdsFromImport.contains(t.getType()));
+
+        importModel.getEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedEntries().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedFalsePositives().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+        importModel.getDeletedFalseRecommendations().keySet().removeIf(systemManagedTypesIdsFromImport::contains);
+    }
 
     private void enrichObservation(EntityTypeImportModel importModel) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java

@@ -36,8 +36,6 @@ public class ManualForceRedactionEntity implements IBaseAnnotation {
     private OffsetDateTime softDeletedTime;
     @Column
     private int page;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java

@@ -39,8 +39,6 @@ public class ManualLegalBasisChangeEntity implements IBaseAnnotation {
     private OffsetDateTime softDeletedTime;
     @Column
     private int page;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRecategorizationEntity.java

@@ -54,8 +54,6 @@ public class ManualRecategorizationEntity implements IBaseAnnotation {
     private String section;
     @Column
     private String value;
-    @Column
-    private String basedOnDictAnnotationId;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java

@@ -61,9 +61,6 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation {
     @Column
     private boolean addToAllDossiers;
 
-    @Column
-    private String basedOnDictAnnotationId;
-
     @ElementCollection
     @Fetch(value = FetchMode.SUBSELECT)
     private Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/AppVersionEntity.java

@@ -0,0 +1,39 @@
+package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
+
+import java.time.OffsetDateTime;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.IdClass;
+import jakarta.persistence.Table;
+import jakarta.validation.constraints.NotNull;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Entity
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@IdClass(AppVersionEntityKey.class)
+@Table(name = "app_version_history")
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class AppVersionEntity {
+
+    @Id
+    @Column
+    @NotNull String appVersion;
+
+    @Id
+    @Column
+    @NotNull String layoutParserVersion;
+
+    @Column
+    @NotNull OffsetDateTime date;
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/AppVersionEntityKey.java

@@ -0,0 +1,27 @@
+package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
+
+import java.io.Serializable;
+
+import jakarta.persistence.Column;
+import jakarta.validation.constraints.NotNull;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class AppVersionEntityKey implements Serializable {
+
+    @Column
+    @NotNull String appVersion;
+
+    @Column
+    @NotNull String layoutParserVersion;
+
+}
+
+

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java

@@ -49,8 +49,14 @@ public class TypeEntity {
     @Column(length = 4000)
     private String description;
     @Column
+    private boolean aiCreationEnabled;
+    @Column(length = 4000)
+    private String aiDescription;
+    @Column
     private long version;
     @Column
+    private long aiCreationVersion;
+    @Column
     private boolean addToDictionaryAction;
     @Column
     private boolean hasDictionary;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeConfigEntity.java

@@ -30,6 +30,10 @@ public class DossierAttributeConfigEntity {
     @Column
     private boolean editable;
     @Column
+    private boolean filterable;
+    @Column
+    private boolean displayedInDossierList;
+    @Column
     private String placeholder;
 
     @Column

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeConfigEntity.java

@@ -41,6 +41,8 @@ public class FileAttributeConfigEntity {
     @Column
     private String placeholder;
     @Column
+    private boolean includeInCsvExport;
+    @Column
     @Enumerated(EnumType.STRING)
     @Builder.Default
     private FileAttributeType type = FileAttributeType.TEXT;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java

@@ -9,7 +9,9 @@ import java.util.Set;
 import org.hibernate.annotations.Fetch;
 import org.hibernate.annotations.FetchMode;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
 import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
 
@@ -21,8 +23,10 @@ import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
 import jakarta.persistence.Enumerated;
 import jakarta.persistence.FetchType;
+import jakarta.persistence.ForeignKey;
 import jakarta.persistence.Id;
 import jakarta.persistence.JoinColumn;
+import jakarta.persistence.ManyToOne;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
@@ -73,6 +77,9 @@ public class FileEntity {
     @Column
     private OffsetDateTime lastLayoutProcessed;
 
+    @Column
+    private String layoutParserVersion;
+
     @Column
     private OffsetDateTime lastIndexed;
 
@@ -109,6 +116,9 @@ public class FileEntity {
     @Column
     private long dictionaryVersion;
 
+    @Column
+    private long aiCreationVersion;
+
     @Column
     private long rulesVersion;
 
@@ -201,6 +211,10 @@ public class FileEntity {
     @Column
     private OffsetDateTime errorTimestamp;
 
+    @Column
+    @Enumerated(EnumType.STRING)
+    private ErrorCode errorCode;
+
     @ElementCollection(fetch = FetchType.EAGER)
     private List<FileEntityComponentMappingVersionEntity> componentMappingVersions;
 
@@ -210,6 +224,11 @@ public class FileEntity {
     @Column
     private boolean protobufMigrationDone;
 
+    @ManyToOne(fetch = FetchType.EAGER)
+    @JoinColumn(name = "last_download", referencedColumnName = "storage_id", foreignKey = @ForeignKey(name = "fk_file_last_download"))
+    private DownloadStatusEntity lastDownload;
+
+
     public OffsetDateTime getLastOCRTime() {
 
         return this.ocrStartTime;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java

@@ -43,6 +43,7 @@ import lombok.experimental.FieldDefaults;
 public class DownloadStatusEntity {
 
     @Id
+    @Column(name = "storage_id")
     String storageId;
     @Column
     String uuid;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/migration/SaasMigrationStatusEntity.java

@@ -1,35 +0,0 @@
-package com.iqser.red.service.persistence.management.v1.processor.entity.migration;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
-
-import jakarta.persistence.*;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@Entity
-@Table(name = "saas_migration_status")
-public class SaasMigrationStatusEntity {
-
-    @Id
-    private String fileId;
-
-    @Column
-    private String dossierId;
-
-    @Column
-    @Enumerated(EnumType.STRING)
-    private SaasMigrationStatus status;
-
-    @Column
-    private Integer processingErrorCounter;
-
-    @Column
-    private String errorCause;
-
-}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/AppVersionTracker.java

@@ -0,0 +1,61 @@
+package com.iqser.red.service.persistence.management.v1.processor.lifecycle;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AppVersionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+import com.knecon.fforesight.tenantcommons.TenantProvider;
+
+import jakarta.validation.ConstraintViolationException;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class AppVersionTracker {
+
+    @Value("${BACKEND_APP_VERSION:}")
+    private String appVersion;
+
+    @Value("${LAYOUT_PARSER_VERSION:}")
+    private String layoutParserVersion;
+
+    private final AppVersionPersistenceService appVersionPersistenceService;
+    private final TenantProvider tenantProvider;
+
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void trackAppVersion() {
+
+        tenantProvider.getTenants()
+                .forEach(tenant -> {
+
+                    if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
+                        return;
+                    }
+
+                    if (appVersion.isBlank() || layoutParserVersion.isBlank()) {
+                        log.info("No app version or layout parser version was provided. Version tracking skipped. ");
+                        return;
+                    }
+
+                    TenantContext.setTenantId(tenant.getTenantId());
+
+                    try {
+                        if (appVersionPersistenceService.insertIfNotExists(appVersion, layoutParserVersion)) {
+                            log.info("Started with new app version {} / layout parser version {}.", appVersion, layoutParserVersion);
+                        }
+                    } catch (ConstraintViolationException cve) {
+                        log.error("Validation failed for app version {} / layout parser version {}.", appVersion, layoutParserVersion, cve);
+                    } catch (Exception e) {
+                        log.error("Failed to track app version {} / layout parser version {}.", appVersion, layoutParserVersion, e);
+                    }
+                });
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/LegalBasisMappingService.java

@@ -0,0 +1,148 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
+
+import org.springframework.stereotype.Service;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+public class LegalBasisMappingService {
+
+    private final HashFunction hashFunction;
+    private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
+
+
+    public LegalBasisMappingService(LegalBasisMappingPersistenceService legalBasisMappingPersistenceService) {
+
+        this.hashFunction = Hashing.murmur3_128();
+        this.legalBasisMappingPersistenceService = legalBasisMappingPersistenceService;
+    }
+
+
+    public TechnicalNameResult processLegalBasisAndEntityLogLegalBasisList(String currentLegalBasis,
+                                                                           Map<String, String> reasonToTechnicalNameMap,
+                                                                           List<EntityLogLegalBasis> legalBasisList) {
+
+        TechnicalNameResult result = computeTechnicalName(currentLegalBasis, reasonToTechnicalNameMap);
+
+        if (result == null) {
+            return null;
+        }
+
+        addToLegalBasisList(result.technicalName(), currentLegalBasis, legalBasisList);
+
+        return result;
+    }
+
+
+    public TechnicalNameResult processLegalBasis(String currentLegalBasis, Map<String, String> reasonToTechnicalNameMap) {
+
+        return computeTechnicalName(currentLegalBasis, reasonToTechnicalNameMap);
+    }
+
+
+    private TechnicalNameResult computeTechnicalName(String currentLegalBasis, Map<String, String> reasonToTechnicalNameMap) {
+
+        if (currentLegalBasis == null || currentLegalBasis.isEmpty()) {
+            return null;
+        }
+
+        if (reasonToTechnicalNameMap.containsValue(currentLegalBasis)) {
+            return new TechnicalNameResult(currentLegalBasis, false);
+        }
+
+        if (reasonToTechnicalNameMap.containsKey(currentLegalBasis)) {
+            String technicalName = reasonToTechnicalNameMap.get(currentLegalBasis);
+            return new TechnicalNameResult(technicalName, false);
+        }
+
+        String technicalName = hashFunction.hashString(currentLegalBasis, StandardCharsets.UTF_8).toString();
+
+        return new TechnicalNameResult(technicalName, true);
+    }
+
+
+    public String processTextAndEntityLogLegalBasisList(String text, Map<String, String> reasonToTechnicalNameMap, List<EntityLogLegalBasis> legalBasisList) {
+
+        if (text == null || text.isEmpty()) {
+            return text;
+        }
+
+        String updatedValue = text;
+        boolean updated = false;
+
+        for (Map.Entry<String, String> entry : reasonToTechnicalNameMap.entrySet()) {
+            String reason = entry.getKey();
+            String technicalName = entry.getValue();
+
+            if (updatedValue.contains(reason)) {
+                updatedValue = updatedValue.replace(reason, technicalName);
+                addToLegalBasisList(technicalName, reason, legalBasisList);
+                updated = true;
+            }
+        }
+
+        if (!updated) {
+            String technicalName = hashFunction.hashString(text, StandardCharsets.UTF_8).toString();
+            updatedValue = technicalName;
+            addToLegalBasisList(technicalName, text, legalBasisList);
+        }
+
+        return updatedValue;
+    }
+
+
+    private void addToLegalBasisList(String technicalName, String reason, List<EntityLogLegalBasis> legalBasisList) {
+
+        boolean exists = legalBasisList.stream()
+                .anyMatch(lb -> lb.getTechnicalName().equals(technicalName));
+
+        if (!exists) {
+            EntityLogLegalBasis newLegalBasis = createEntityLogLegalBasis(technicalName, reason);
+            legalBasisList.add(newLegalBasis);
+        }
+    }
+
+
+    private EntityLogLegalBasis createEntityLogLegalBasis(String technicalName, String reason) {
+
+        EntityLogLegalBasis newLegalBasis = new EntityLogLegalBasis();
+        newLegalBasis.setTechnicalName(technicalName);
+        newLegalBasis.setReason(reason);
+        newLegalBasis.setName("Generated Legal Basis");
+        newLegalBasis.setDescription("");
+        return newLegalBasis;
+    }
+
+
+    public Map<String, String> getReasonToTechnicalNameMap(String dossierTemplateId) {
+
+        List<LegalBasisEntity> legalBasisMappings = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId);
+
+        if (legalBasisMappings == null) {
+            legalBasisMappings = new ArrayList<>();
+        }
+
+        Map<String, String> reasonToTechnicalNameMap = new HashMap<>();
+        for (LegalBasisEntity lbEntity : legalBasisMappings) {
+            reasonToTechnicalNameMap.put(lbEntity.getReason(), lbEntity.getTechnicalName());
+        }
+        return reasonToTechnicalNameMap;
+
+    }
+
+
+    public record TechnicalNameResult(String technicalName, boolean hashed) {
+
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java

@@ -1,170 +0,0 @@
-package com.iqser.red.service.persistence.management.v1.processor.migration;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
-import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
-
-import jakarta.transaction.Transactional;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Service;
-
-@Slf4j
-@Service
-@Transactional
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class SaasAnnotationIdMigrationService {
-
-    ManualRedactionRepository manualRedactionRepository;
-    RemoveRedactionRepository removeRedactionRepository;
-    ForceRedactionRepository forceRedactionRepository;
-    ResizeRedactionRepository resizeRedactionRepository;
-    RecategorizationRepository recategorizationRepository;
-    LegalBasisChangeRepository legalBasisChangeRepository;
-    CommentRepository commentRepository;
-    FileRepository fileRepository;
-
-
-    public int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = manualRedactionRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualRedactionEntryEntity.class);
-            newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class));
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-            newEntry.setId(newAnnotationEntityId);
-
-            manualRedactionRepository.deleteById(oldAnnotationEntityId);
-            manualRedactionRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = removeRedactionRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), IdRemovalEntity.class);
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-            newEntry.setId(newAnnotationEntityId);
-
-            removeRedactionRepository.deleteById(oldAnnotationEntityId);
-            removeRedactionRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = forceRedactionRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualForceRedactionEntity.class);
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-            newEntry.setId(newAnnotationEntityId);
-
-            forceRedactionRepository.deleteById(oldAnnotationEntityId);
-            forceRedactionRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = resizeRedactionRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualResizeRedactionEntity.class);
-            newEntry.setId(newAnnotationEntityId);
-            newEntry.setPositions(MagicConverter.convert(oldEntry.get().getPositions(), RectangleEntity.class));
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-
-            resizeRedactionRepository.deleteById(oldAnnotationEntityId);
-            resizeRedactionRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = recategorizationRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualRecategorizationEntity.class);
-            newEntry.setId(newAnnotationEntityId);
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-
-            recategorizationRepository.deleteById(oldAnnotationEntityId);
-            recategorizationRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-
-        if (oldAnnotationEntityId.equals(newAnnotationEntityId)) {
-            return 0;
-        }
-        var oldEntry = legalBasisChangeRepository.findById(oldAnnotationEntityId);
-        if (oldEntry.isPresent()) {
-
-            var newEntry = MagicConverter.convert(oldEntry.get(), ManualLegalBasisChangeEntity.class);
-            newEntry.setId(newAnnotationEntityId);
-            newEntry.setFileStatus(fileRepository.findById(oldAnnotationEntityId.getFileId())
-                                           .get());
-
-            legalBasisChangeRepository.deleteById(oldAnnotationEntityId);
-            legalBasisChangeRepository.save(newEntry);
-            return 1;
-        }
-        return 0;
-    }
-
-
-    public int updateCommentIds(String fileId, String key, String value) {
-
-        if (key.equals(value)) {
-            return 0;
-        }
-        return commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value);
-    }
-
-}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java

@@ -1,83 +0,0 @@
-package com.iqser.red.service.persistence.management.v1.processor.migration;
-
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-
-import org.springframework.stereotype.Service;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
-import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
-
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class SaasMigrationManualChangesUpdateService {
-
-    private final AddRedactionPersistenceService addRedactionPersistenceService;
-
-    private final HashFunction hashFunction = Hashing.murmur3_128();
-
-
-    public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId) {
-
-        var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly());
-        for (var unprocessedManualAdd : unprocessedManualAdds) {
-
-            if (!unprocessedManualAdd.getDictionaryEntryType().equals(DictionaryEntryType.ENTRY)) {
-                continue;
-            }
-
-            if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) {
-                // copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table.
-                // It has no functionality, its only there, such that the UI can show a pending change.
-                ManualRedactionEntryEntity pendingDictAdd = new ManualRedactionEntryEntity(buildSecondaryId(unprocessedManualAdd.getId(), fileId),
-                                                                                           unprocessedManualAdd.getUser(),
-                                                                                           unprocessedManualAdd.getTypeId(),
-                                                                                           unprocessedManualAdd.getValue(),
-                                                                                           unprocessedManualAdd.getReason(),
-                                                                                           unprocessedManualAdd.getLegalBasis(),
-                                                                                           unprocessedManualAdd.getSection(),
-                                                                                           unprocessedManualAdd.isRectangle(),
-                                                                                           unprocessedManualAdd.isAddToDictionary(),
-                                                                                           unprocessedManualAdd.isAddToAllDossiers(),
-                                                                                           unprocessedManualAdd.isAddToDossierDictionary(),
-                                                                                           DictionaryEntryType.ENTRY,
-                                                                                           unprocessedManualAdd.getRequestDate(),
-                                                                                           null,
-                                                                                           null,
-                                                                                           new ArrayList<>(unprocessedManualAdd.getPositions()),
-                                                                                           unprocessedManualAdd.getFileStatus(),
-                                                                                           unprocessedManualAdd.getTextBefore(),
-                                                                                           unprocessedManualAdd.getTextAfter(),
-                                                                                           unprocessedManualAdd.getSourceId(),
-                                                                                           new HashSet<>(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries()));
-
-                addRedactionPersistenceService.update(pendingDictAdd);
-
-                // change existing dict add to unprocessed manual add. ID must match with prior entry, such that other unprocessed manual changes may be applied to it.
-                unprocessedManualAdd.setAddToDictionary(false);
-                unprocessedManualAdd.setAddToAllDossiers(false);
-                unprocessedManualAdd.setLegalBasis("");
-                unprocessedManualAdd.setTypeIdsOfModifiedDictionaries(Collections.emptySet());
-                unprocessedManualAdd.setDictionaryEntryType(null);
-
-                addRedactionPersistenceService.update(unprocessedManualAdd);
-            }
-        }
-    }
-
-
-    private AnnotationEntityId buildSecondaryId(AnnotationEntityId annotationEntityId, String fileId) {
-
-        return new AnnotationEntityId(hashFunction.hashString(annotationEntityId.getAnnotationId(), StandardCharsets.UTF_8).toString(), fileId);
-    }
-
-}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java

@@ -1,398 +0,0 @@
-package com.iqser.red.service.persistence.management.v1.processor.migration;
-
-import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_REQUEST_QUEUE;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
-import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
-import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
-import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
-import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService;
-import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
-import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
-import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
-import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
-import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
-import com.iqser.red.service.redaction.v1.model.MigrationRequest;
-import com.iqser.red.storage.commons.exception.StorageException;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-import com.knecon.fforesight.tenantcommons.TenantProvider;
-import com.knecon.fforesight.tenantcommons.model.TenantSyncEvent;
-import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class SaasMigrationService implements TenantSyncService {
-
-    AutomaticAnalysisJob automaticAnalysisJob;
-    FileStatusPersistenceService fileStatusPersistenceService;
-    SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
-    DossierService dossierService;
-    ManualRedactionProviderService manualRedactionProviderService;
-    TenantProvider tenantProvider;
-    IndexingService indexingService;
-    LayoutParsingRequestFactory layoutParsingRequestFactory;
-    RabbitTemplate rabbitTemplate;
-    FileManagementServiceSettings settings;
-    StorageService storageService;
-    SaasAnnotationIdMigrationService saasAnnotationIdMigrationService;
-    UncompressedFilesMigrationService uncompressedFilesMigrationService;
-    ManualRedactionService manualRedactionService;
-    CommentService commentService;
-    RankDeDuplicationService rankDeDuplicationService;
-    SaasMigrationManualChangesUpdateService saasMigrationManualChangesUpdateService;
-
-
-    @Override
-    public synchronized void syncTenant(TenantSyncEvent tenantSyncEvent) {
-
-        startMigrationForTenant(tenantSyncEvent.getTenantId());
-    }
-
-    // Persistence-Service needs to be scaled to 1.
-
-
-    public void startMigrationForTenant(String tenantId) {
-
-        // TODO migrate rules.
-        automaticAnalysisJob.stopForTenant(tenantId);
-
-        log.info("Starting uncompressed files migration ...");
-        uncompressedFilesMigrationService.migrateUncompressedFiles(tenantId);
-        log.info("Finished uncompressed files migration ...");
-
-        rankDeDuplicationService.deduplicate();
-        int numberOfFiles = 0;
-
-        var files = saasMigrationStatusPersistenceService.findAll();
-
-        for (var file : files) {
-
-            var dossier = dossierService.getDossierById(file.getDossierId());
-
-            if (dossier.getHardDeletedTime() != null) {
-                if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
-                    saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
-                    continue;
-                } else {
-                    fileStatusPersistenceService.hardDelete(file.getFileId(), dossier.getHardDeletedTime());
-                    saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
-                    continue;
-                }
-            }
-            if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
-                saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
-                continue;
-            }
-
-            if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) {
-                log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus());
-                continue;
-            }
-
-            // delete NER_ENTITIES since offsets depend on old document structure.
-            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(file.getDossierId(), file.getFileId(), FileType.NER_ENTITIES));
-
-            var layoutParsingRequest = layoutParsingRequestFactory.build(file.getDossierId(), file.getFileId(), false);
-
-            rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest);
-
-            numberOfFiles++;
-
-        }
-
-        log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId());
-        if (numberOfFiles == 0) {
-            finalizeMigration();
-        }
-    }
-
-
-    public void startMigrationForFile(String dossierId, String fileId) {
-
-        var dossier = dossierService.getDossierById(dossierId);
-
-        if (dossier.getHardDeletedTime() != null) {
-            if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
-                saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
-                return;
-            } else {
-                fileStatusPersistenceService.hardDelete(fileId, dossier.getHardDeletedTime());
-                saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
-                return;
-            }
-        }
-        if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
-            saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
-            return;
-        }
-
-        log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId);
-        saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
-        var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, false);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE, TenantContext.getTenantId(), layoutParsingRequest);
-    }
-
-
-    public void handleLayoutParsingFinished(String dossierId, String fileId) {
-
-        if (!layoutParsingFilesExist(dossierId, fileId)) {
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Layout parsing files not written!");
-            return;
-        }
-
-        log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
-        saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED);
-
-        if (fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
-            saasMigrationManualChangesUpdateService.convertUnprocessedAddToDictionariesToLocalChanges(fileId);
-        }
-
-        try {
-            indexingService.reindex(dossierId, Set.of(fileId), false);
-
-            String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId();
-
-            rabbitTemplate.convertAndSend(MIGRATION_REQUEST_QUEUE,
-                                          MigrationRequest.builder()
-                                                  .dossierTemplateId(dossierTemplateId)
-                                                  .dossierId(dossierId)
-                                                  .fileId(fileId)
-                                                  .fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED))
-                                                  .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
-                                                  .entitiesWithComments(commentService.getCommentCounts(fileId).keySet())
-                                                  .build());
-        } catch (Exception e) {
-            log.error("Queuing of entityLog migration failed with {}", e.getMessage());
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage()));
-        }
-
-    }
-
-
-    private boolean layoutParsingFilesExist(String dossierId, String fileId) {
-
-        return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) //
-               && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) //
-               && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) //
-               && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION));
-    }
-
-
-    public void handleEntityLogMigrationFinished(String dossierId, String fileId) {
-
-        if (!entityLogMigrationFilesExist(dossierId, fileId)) {
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, "Migration Files not written!");
-            return;
-        }
-        saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED);
-
-        log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
-        migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(dossierId, fileId);
-    }
-
-
-    private boolean entityLogMigrationFilesExist(String dossierId, String fileId) {
-
-        return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists(
-                TenantContext.getTenantId(),
-                StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS));
-    }
-
-
-    public void handleError(String dossierId, String fileId, String errorCause, String retryExchange) {
-
-        var migrationEntry = saasMigrationStatusPersistenceService.findById(fileId);
-        Integer numErrors = migrationEntry.getProcessingErrorCounter();
-        if (numErrors != null && numErrors <= settings.getMaxErrorRetries()) {
-            saasMigrationStatusPersistenceService.updateErrorCounter(fileId, numErrors + 1, errorCause);
-            rabbitTemplate.convertAndSend(retryExchange, TenantContext.getTenantId(), MigrationRequest.builder().dossierId(dossierId).fileId(fileId).build());
-            log.error("Retrying error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause);
-        } else {
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, errorCause);
-            log.error("Error during saas migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, errorCause);
-        }
-    }
-
-
-    public void requeueErrorFiles() {
-
-        automaticAnalysisJob.stopForTenant(TenantContext.getTenantId());
-        saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR)
-                .forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId()));
-
-    }
-
-
-    private void migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(String dossierId, String fileId) {
-
-        MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
-        Map<String, String> oldToNewMapping = migratedIds.buildOldToNewMapping();
-        updateAnnotationIds(dossierId, fileId, oldToNewMapping);
-        List<String> forceRedactionIdsToDelete = migratedIds.getForceRedactionIdsToDelete();
-        softDeleteForceRedactions(fileId, forceRedactionIdsToDelete);
-        log.info("Soft-deleted force redactions.");
-        List<ManualRedactionEntry> manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd();
-        int count = addManualRedactionEntries(manualRedactionEntriesToAdd);
-        log.info("Added {} additional manual entries.", count);
-        deleteSectionGridAndNerEntitiesFiles(dossierId, fileId);
-        saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
-
-        log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
-        finalizeMigration(); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
-    }
-
-
-    private void deleteSectionGridAndNerEntitiesFiles(String dossierId, String fileId) {
-
-        try {
-            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
-        } catch (StorageObjectDoesNotExist e) {
-            log.info("No sectiongrid found for {}, {}, ignoring....", dossierId, fileId);
-        }
-
-        try {
-            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
-        } catch (StorageObjectDoesNotExist e) {
-            log.info("No ner entities file found for {}, {}, ignoring....", dossierId, fileId);
-        }
-    }
-
-
-    private void softDeleteForceRedactions(String fileId, List<String> forceRedactionIdsToDelete) {
-
-        manualRedactionService.softDeleteForceRedactions(fileId, forceRedactionIdsToDelete);
-    }
-
-
-    private int addManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
-
-        manualRedactionEntriesToAdd.forEach(add -> {
-            if (add.getSection() != null && add.getSection().length() > 254) {
-                add.setSection(add.getSection().substring(0, 254));
-            }
-        });
-
-        return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true);
-    }
-
-
-    public void revertMigrationForFile(String dossierId, String fileId) {
-
-        log.info("Reverting Migration for dossierId {} and fileId {}", dossierId, fileId);
-        MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
-        Map<String, String> newToOldMapping = migratedIds.buildNewToOldMapping();
-        updateAnnotationIds(dossierId, fileId, newToOldMapping);
-        deleteManualRedactionEntries(migratedIds.getManualRedactionEntriesToAdd());
-        undeleteForceRedactions(fileId, migratedIds.getForceRedactionIdsToDelete());
-        saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
-    }
-
-
-    private void undeleteForceRedactions(String fileId, List<String> forceRedactionIdsToDelete) {
-
-        manualRedactionService.undeleteForceRedactions(fileId, forceRedactionIdsToDelete);
-
-    }
-
-
-    private void deleteManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
-
-        manualRedactionService.deleteManualRedactionEntries(manualRedactionEntriesToAdd);
-    }
-
-
-    private void updateAnnotationIds(String dossierId, String fileId, Map<String, String> idMapping) {
-
-        try {
-            updateAnnotationIds(fileId, idMapping);
-        } catch (Exception e) {
-            String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s",
-                                           TenantContext.getTenantId(),
-                                           dossierId,
-                                           fileId,
-                                           e.getMessage());
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message);
-            log.error(message);
-            throw e;
-        }
-    }
-
-
-    private void finalizeMigration() {
-
-        if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
-//            automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
-            tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build());
-            log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
-        }
-    }
-
-
-    public void updateAnnotationIds(String fileId, Map<String, String> idMapping) {
-
-        AtomicInteger numUpdates = new AtomicInteger(0);
-        AtomicInteger numCommentUpdates = new AtomicInteger(0);
-        idMapping.forEach((key, value) -> {
-            AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key);
-            AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value);
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numCommentUpdates.getAndAdd(saasAnnotationIdMigrationService.updateCommentIds(fileId, key, value));
-        });
-        log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId);
-    }
-
-
-    private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
-
-        return new AnnotationEntityId(annotationId, fileId);
-    }
-
-
-    private MigratedIds getMigratedIds(String dossierId, String fileId) {
-
-        try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS), MigratedIds.class);
-        } catch (StorageObjectDoesNotExist e) {
-            throw new NotFoundException(String.format("MigratedIds does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId));
-        } catch (StorageException e) {
-            throw new InternalServerErrorException(e.getMessage());
-        }
-    }
-
-}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/liquibase/TechnicalNameChange.java

@@ -0,0 +1,259 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.liquibase;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+
+import liquibase.change.custom.CustomTaskChange;
+import liquibase.database.Database;
+import liquibase.database.jvm.JdbcConnection;
+import liquibase.exception.CustomChangeException;
+import liquibase.exception.ValidationErrors;
+import liquibase.resource.ResourceAccessor;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@SuppressWarnings("PMD.CloseResource") // we do not want to close the underlying database connection ;)
+public class TechnicalNameChange implements CustomTaskChange {
+
+    // there is a bug in our liquibase version which causes custom task changes to be run twice
+    // that bug got fixed in liquibase 4.25.1
+    // when we upgrade the dependency this can then be removed
+    @Setter
+    private static boolean skipExecution;
+
+
+    @Override
+    public void execute(Database database) throws CustomChangeException {
+
+        if (skipExecution) {
+            return;
+        }
+
+        Connection connection;
+
+        try {
+            connection = ((JdbcConnection) database.getConnection()).getUnderlyingConnection();
+
+            String[] tables = {"manual_legal_basis_change", "manual_force_redaction", "manual_recategorization", "manual_redaction"};
+
+            HashFunction hashFunction = Hashing.murmur3_128();
+
+            Set<String> allFileIds = collectAllFileIds(connection, tables);
+
+            if (allFileIds.isEmpty()) {
+                log.info("No rows updated because no files with manual redactions are present.");
+                return;
+            }
+
+            Map<String, String> fileIdToDossierTemplateIdMap = buildFileIdToDossierTemplateIdMap(connection, allFileIds);
+
+            Map<String, String> reasonDossierTemplateIdToTechnicalNameMap = buildReasonDossierTemplateIdToTechnicalNameMap(connection);
+
+            int totalRowsUpdatedWithTechnicalName = 0;
+            int totalRowsHashed = 0;
+
+            for (String tableName : tables) {
+                int[] counts = updateLegalBasisForTable(connection, tableName, hashFunction, fileIdToDossierTemplateIdMap, reasonDossierTemplateIdToTechnicalNameMap);
+                int rowsUpdatedWithTechnicalName = counts[0];
+                int rowsHashed = counts[1];
+
+                log.info("Table '{}': {} rows updated with technical_name, {} rows hashed.", tableName, rowsUpdatedWithTechnicalName, rowsHashed);
+
+                totalRowsUpdatedWithTechnicalName += rowsUpdatedWithTechnicalName;
+                totalRowsHashed += rowsHashed;
+            }
+
+            log.info("Total rows updated with technical_name: {}", totalRowsUpdatedWithTechnicalName);
+            log.info("Total rows hashed: {}", totalRowsHashed);
+
+        } catch (Exception e) {
+            throw new CustomChangeException("Error applying technical name change", e);
+        }
+        skipExecution = true;
+    }
+
+
+    private Set<String> collectAllFileIds(Connection connection, String[] tables) throws SQLException {
+
+        Set<String> fileIds = new HashSet<>();
+
+        for (String tableName : tables) {
+            String selectSql = "SELECT DISTINCT file_id FROM " + tableName + " WHERE file_id IS NOT NULL";
+            try (PreparedStatement select = connection.prepareStatement(selectSql); ResultSet resultSet = select.executeQuery()) {
+                while (resultSet.next()) {
+                    String fileId = resultSet.getString("file_id");
+                    fileIds.add(fileId);
+                }
+            }
+        }
+
+        return fileIds;
+    }
+
+
+    private Map<String, String> buildFileIdToDossierTemplateIdMap(Connection connection, Set<String> fileIds) throws SQLException {
+
+        Map<String, String> map = new HashMap<>();
+
+        if (!fileIds.isEmpty()) {
+
+            StringBuilder sqlBuilder = new StringBuilder();
+            sqlBuilder.append("SELECT f.id AS file_id, d.dossier_template_id ");
+            sqlBuilder.append("FROM file f ");
+            sqlBuilder.append("JOIN dossier d ON f.dossier_id = d.id ");
+            sqlBuilder.append("WHERE f.id IN (");
+
+            String placeholders = String.join(",", Collections.nCopies(fileIds.size(), "?"));
+            sqlBuilder.append(placeholders);
+            sqlBuilder.append(")");
+
+            String selectSql = sqlBuilder.toString();
+            try (PreparedStatement select = connection.prepareStatement(selectSql)) {
+                int index = 1;
+                for (String fileId : fileIds) {
+                    select.setString(index++, fileId);
+                }
+                try (ResultSet rs = select.executeQuery()) {
+                    while (rs.next()) {
+                        String fileId = rs.getString("file_id");
+                        String dossierTemplateId = rs.getString("dossier_template_id");
+                        map.put(fileId, dossierTemplateId);
+                    }
+                }
+            }
+        }
+
+        return map;
+    }
+
+
+    private Map<String, String> buildReasonDossierTemplateIdToTechnicalNameMap(Connection connection) throws SQLException {
+
+        Map<String, String> map = new HashMap<>();
+
+        String selectSql = "SELECT reason, legal_basis_mapping_entity_dossier_template_id, technical_name FROM legal_basis_mapping_entity_legal_basis";
+        try (PreparedStatement select = connection.prepareStatement(selectSql); ResultSet resultSet = select.executeQuery()) {
+            while (resultSet.next()) {
+                String reason = resultSet.getString("reason");
+                String dossierTemplateId = resultSet.getString("legal_basis_mapping_entity_dossier_template_id");
+                String technicalName = resultSet.getString("technical_name");
+
+                if (reason != null && dossierTemplateId != null && technicalName != null) {
+                    String key = getKey(reason, dossierTemplateId);
+                    map.put(key, technicalName);
+                }
+            }
+        }
+
+        return map;
+    }
+
+
+    private int[] updateLegalBasisForTable(Connection connection,
+                                           String tableName,
+                                           HashFunction hashFunction,
+                                           Map<String, String> fileIdToDossierTemplateIdMap,
+                                           Map<String, String> reasonDossierTemplateIdToTechnicalNameMap) throws SQLException {
+
+        PreparedStatement select = null;
+        PreparedStatement update = null;
+        ResultSet resultSet = null;
+
+        int rowsUpdatedWithTechnicalName = 0;
+        int rowsHashed = 0;
+
+        try {
+            String selectSql = "SELECT annotation_id, file_id, legal_basis FROM " + tableName + " WHERE legal_basis IS NOT NULL AND TRIM(legal_basis) <> '';";
+            select = connection.prepareStatement(selectSql);
+            resultSet = select.executeQuery();
+
+            String updateSql = "UPDATE " + tableName + " SET legal_basis = ? WHERE annotation_id = ? AND file_id = ?";
+            update = connection.prepareStatement(updateSql);
+
+            while (resultSet.next()) {
+                String annotationId = resultSet.getString("annotation_id");
+                String fileId = resultSet.getString("file_id");
+                String originalValue = resultSet.getString("legal_basis");
+
+                if (originalValue == null) {
+                    continue;
+                }
+
+                String dossierTemplateId = fileIdToDossierTemplateIdMap.get(fileId);
+
+                String newValue = null;
+
+                if (dossierTemplateId != null) {
+                    String key = getKey(originalValue, dossierTemplateId);
+                    newValue = reasonDossierTemplateIdToTechnicalNameMap.get(key);
+                }
+
+                if (newValue == null || newValue.trim().isEmpty()) {
+                    newValue = hashFunction.hashString(originalValue, StandardCharsets.UTF_8).toString();
+                    rowsHashed++;
+                } else {
+                    rowsUpdatedWithTechnicalName++;
+                }
+
+                update.setString(1, newValue);
+                update.setString(2, annotationId);
+                update.setString(3, fileId);
+                update.executeUpdate();
+            }
+
+        } finally {
+            if (resultSet != null) {
+                resultSet.close();
+            }
+            if (select != null) {
+                select.close();
+            }
+            if (update != null) {
+                update.close();
+            }
+        }
+
+        return new int[]{rowsUpdatedWithTechnicalName, rowsHashed};
+    }
+
+
+    private static String getKey(String originalValue, String dossierTemplateId) {
+
+        return originalValue + "|" + dossierTemplateId;
+    }
+
+
+    @Override
+    public String getConfirmationMessage() {
+
+        return "Technical name change applied to legalbasis fields in specified tables.";
+    }
+
+
+    @Override
+    public void setUp() {}
+
+
+    @Override
+    public void setFileOpener(ResourceAccessor resourceAccessor) {}
+
+
+    @Override
+    public ValidationErrors validate(Database database) {
+
+        return new ValidationErrors();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V10ReduceTextFileSizeMigration.java

@@ -24,7 +24,7 @@ import lombok.extern.slf4j.Slf4j;
 @Setter
 @Service
 @SuppressWarnings("PMD")
-public class ReduceTextFileSizeMigration10 extends Migration {
+public class V10ReduceTextFileSizeMigration extends Migration {
 
     private static final String NAME = "Reduce TEXT filesize migration";
     private static final long VERSION = 10;
@@ -42,7 +42,7 @@ public class ReduceTextFileSizeMigration10 extends Migration {
     private FileManagementStorageService fileManagementStorageService;
 
 
-    public ReduceTextFileSizeMigration10() {
+    public V10ReduceTextFileSizeMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V13MissingFileSizeMigration.java

@@ -17,7 +17,7 @@ import java.util.List;
 @Slf4j
 @Setter
 @Service
-public class MissingFileSizeMigration13 extends Migration {
+public class V13MissingFileSizeMigration extends Migration {
 
     private static final String NAME = "Add missing file sizes";
     private static final long VERSION = 13;
@@ -29,7 +29,7 @@ public class MissingFileSizeMigration13 extends Migration {
     private FileManagementStorageService fileManagementStorageService;
 
 
-    public MissingFileSizeMigration13() {
+    public V13MissingFileSizeMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V14FixDossierDictionaryEntryInRedactionLog.java

@@ -10,13 +10,13 @@ import org.springframework.stereotype.Service;
 @Slf4j
 @Setter
 @Service
-public class FixDossierDictionaryEntryInRedactionLog14 extends Migration {
+public class V14FixDossierDictionaryEntryInRedactionLog extends Migration {
 
     private static final String NAME = "Fix dossier dictionary entries in redactionLog for non dossier dictionaries";
     private static final long VERSION = 14;
 
 
-    public FixDossierDictionaryEntryInRedactionLog14() {
+    public V14FixDossierDictionaryEntryInRedactionLog() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V15ManualRedactionTypeRenameMigration.java

@@ -28,7 +28,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class ManualRedactionTypeRenameMigration15 extends Migration {
+public class V15ManualRedactionTypeRenameMigration extends Migration {
 
     @Autowired
     private FileStatusPersistenceService fileStatusPersistenceService;
@@ -40,7 +40,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration {
     private ManualRedactionService manualRedactionService;
 
 
-    public ManualRedactionTypeRenameMigration15() {
+    public V15ManualRedactionTypeRenameMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V16RankDeDuplicationMigration.java

@@ -12,13 +12,13 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class RankDeDuplicationMigration16 extends Migration {
+public class V16RankDeDuplicationMigration extends Migration {
 
     @Autowired
     private RankDeDuplicationService rankDeDuplicationService;
 
 
-    public RankDeDuplicationMigration16() {
+    public V16RankDeDuplicationMigration() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V17MigrateImportedRedactionsFiles.java

@@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class MigrateImportedRedactionsFiles17 extends Migration {
+public class V17MigrateImportedRedactionsFiles extends Migration {
 
     @Autowired
     private FileStatusPersistenceService fileStatusPersistenceService;
@@ -34,7 +34,7 @@ public class MigrateImportedRedactionsFiles17 extends Migration {
     private static final long VERSION = 17;
 
 
-    public MigrateImportedRedactionsFiles17() {
+    public V17MigrateImportedRedactionsFiles() {
 
         super(NAME, VERSION);
     }
@@ -48,7 +48,7 @@ public class MigrateImportedRedactionsFiles17 extends Migration {
 
             if (fileManagementStorageService.objectExists(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS)) {
 
-                com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions oldImportedRedactions = fileManagementStorageService.getImportedRedactions(
+                com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions oldImportedRedactions = fileManagementStorageService.getOldImportedRedactions(
                         file.getDossierId(),
                         file.getId());
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V18StorageToMongoMigration.java

@@ -11,7 +11,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class StorageToMongoMigration18 extends Migration {
+public class V18StorageToMongoMigration extends Migration {
 
     private final StorageToMongoCopyService storageToMongoCopyService;
 
@@ -19,7 +19,7 @@ public class StorageToMongoMigration18 extends Migration {
     private static final long VERSION = 18;
 
 
-    public StorageToMongoMigration18(StorageToMongoCopyService storageToMongoCopyService) {
+    public V18StorageToMongoMigration(StorageToMongoCopyService storageToMongoCopyService) {
 
         super(NAME, VERSION);
         this.storageToMongoCopyService = storageToMongoCopyService;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V19AddGraphicDictionaryType.java

@@ -16,7 +16,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class AddGraphicDictionaryType19 extends Migration {
+public class V19AddGraphicDictionaryType extends Migration {
 
     @Autowired
     private DossierTemplatePersistenceService dossierTemplatePersistenceService;
@@ -31,7 +31,7 @@ public class AddGraphicDictionaryType19 extends Migration {
     private static final long VERSION = 19;
 
 
-    public AddGraphicDictionaryType19() {
+    public V19AddGraphicDictionaryType() {
 
         super(NAME, VERSION);
     }
@@ -60,6 +60,8 @@ public class AddGraphicDictionaryType19 extends Migration {
                                                              false,
                                                              "Empty dictionary used to configure graphic colors.",
                                                              false,
+                                                             null,
+                                                             false,
                                                              "Graphic",
                                                              null,
                                                              true,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V20DocumineLayoutRewriteMigration.java

@@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
@@ -16,7 +17,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class DocumineLayoutRewriteMigration20 extends Migration  {
+public class V20DocumineLayoutRewriteMigration extends Migration  {
 
     private static final String NAME = "Reanalyse layout for not approved Documine files";
     private static final long VERSION = 20;
@@ -30,13 +31,11 @@ public class DocumineLayoutRewriteMigration20 extends Migration  {
     @Autowired
     private FileStatusPersistenceService fileStatusPersistenceService;
 
-
-    @Value("${application.type}")
-    private String applicationType;
+    @Autowired
+    private CurrentApplicationTypeProvider currentApplicationTypeProvider;
 
 
-
-    public DocumineLayoutRewriteMigration20() {
+    public V20DocumineLayoutRewriteMigration() {
 
         super(NAME, VERSION);
     }
@@ -45,7 +44,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration  {
     @Override
     protected void migrate() {
 
-        if(!applicationType.equalsIgnoreCase("DocuMine")){
+        if (!currentApplicationTypeProvider.isDocuMine()) {
             log.info("Skipping DocumineLayoutRewriteMigration20 as application type is not DocuMine!!!");
             return;
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V21ComponentOverridesMigration.java

@@ -3,23 +3,21 @@ package com.iqser.red.service.persistence.management.v1.processor.migration.migr
 import java.util.List;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.pdftron.redaction.v1.api.model.ApplicationType;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
 import com.iqser.red.service.persistence.management.v1.processor.migration.StorageToMongoCopyService;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentDefinitionService;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest;
 
-import lombok.experimental.NonFinal;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
-public class ComponentOverridesMigration21 extends Migration {
+public class V21ComponentOverridesMigration extends Migration {
 
     private static final String NAME = "Migrate component overrides to mongoDB and create component definitions";
     private static final long VERSION = 21;
@@ -116,19 +114,17 @@ public class ComponentOverridesMigration21 extends Migration {
                                                                                                                        "Certificate of analysis batch identification",
                                                                                                                        "Certificate of analysis batch identification"));
 
-    @NonFinal
-    @Value("${application.type}")
-    String applicationType;
-
     @Autowired
     ComponentDefinitionService componentDefinitionService;
     @Autowired
     DossierTemplatePersistenceService dossierTemplatePersistenceService;
     @Autowired
     StorageToMongoCopyService storageToMongoCopyService;
+    @Autowired
+    private CurrentApplicationTypeProvider currentApplicationTypeProvider;
 
 
-    public ComponentOverridesMigration21() {
+    public V21ComponentOverridesMigration() {
 
         super(NAME, VERSION);
     }
@@ -137,7 +133,7 @@ public class ComponentOverridesMigration21 extends Migration {
     @Override
     protected void migrate() {
 
-        if (!applicationType.equals("DocuMine")) {
+        if (!currentApplicationTypeProvider.isDocuMine()) {
             log.info("Skipping component migration, due to application type not being equal to DOCUMINE!");
         }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V22DocumineHeadlineDetectionMigration.java

@@ -0,0 +1,69 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Setter
+@Service
+public class V22DocumineHeadlineDetectionMigration extends Migration {
+
+    private static final String NAME = "Reanalyse not approved Documine files after new headline detection";
+    private static final long VERSION = 22;
+
+    @Autowired
+    private FileStatusService fileStatusService;
+
+    @Autowired
+    private DossierPersistenceService dossierPersistenceService;
+
+    @Autowired
+    private FileStatusPersistenceService fileStatusPersistenceService;
+
+    @Autowired
+    private CurrentApplicationTypeProvider currentApplicationTypeProvider;
+
+    public V22DocumineHeadlineDetectionMigration() {
+
+        super(NAME, VERSION);
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Starting migration DocumineHeadlineDetectionMigration22");
+
+        if (!currentApplicationTypeProvider.isDocuMine()) {
+            log.info("Skipping DocumineHeadlineDetectionMigration22 as application type is not DocuMine!");
+            return;
+        }
+
+        var dossiers = dossierPersistenceService.findAllDossiers();
+        dossiers.forEach(dossier -> {
+            if (dossier.getHardDeletedTime() == null) {
+                var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId());
+                log.info("Start migration of dossier {}", dossier.getId());
+                files.forEach(file -> {
+                    if (file.getHardDeletedTime() == null && !file.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
+                        log.info("Set full reanalyse for file {}", file.getId());
+                        fileStatusService.setStatusFullReprocess(dossier.getId(), file.getId(), false, true, false);
+                        log.info("Finished migration of file {}", file.getId());
+                    }
+                });
+                log.info("Finished migration of dossier {}", dossier.getId());
+            }
+        });
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V23ManualChangesReorderingMigration.java

@@ -20,18 +20,18 @@ import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
-public class ManualChangesReorderingMigration28 extends Migration {
+public class V23ManualChangesReorderingMigration extends Migration {
 
     private static final String NAME = "Migration for reordering mixed up manual changes";
-    private static final long VERSION = 28;
+    private static final long VERSION = 23;
     private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
     private final AddRedactionPersistenceService addRedactionPersistenceService;
     private final FileStatusService fileStatusService;
 
 
-    public ManualChangesReorderingMigration28(EntityLogEntryDocumentRepository entityLogEntryDocumentRepository,
-                                              AddRedactionPersistenceService addRedactionPersistenceService,
-                                              FileStatusService fileStatusService) {
+    public V23ManualChangesReorderingMigration(EntityLogEntryDocumentRepository entityLogEntryDocumentRepository,
+                                               AddRedactionPersistenceService addRedactionPersistenceService,
+                                               FileStatusService fileStatusService) {
 
         super(NAME, VERSION);
         this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V24StorageToMongoMigration.java

@@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 @Setter
 @Service
 
-public class StorageToMongoMigration24 extends Migration {
+public class V24StorageToMongoMigration extends Migration {
 
     private final StorageToMongoCopyService storageToMongoCopyService;
     private final MongoTemplate mongoTemplate;
@@ -29,7 +29,7 @@ public class StorageToMongoMigration24 extends Migration {
     private static final long VERSION = 24;
 
 
-    public StorageToMongoMigration24(StorageToMongoCopyService storageToMongoCopyService, MongoTemplate mongoTemplate) {
+    public V24StorageToMongoMigration(StorageToMongoCopyService storageToMongoCopyService, MongoTemplate mongoTemplate) {
 
         super(NAME, VERSION);
         this.storageToMongoCopyService = storageToMongoCopyService;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V25AddDateFormatsToTemplatesMigration.java

@@ -1,20 +1,14 @@
 package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
 
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
 import com.iqser.red.service.persistence.management.v1.processor.service.DefaultDateFormatsProvider;
+import com.iqser.red.service.persistence.management.v1.processor.service.CurrentApplicationTypeProvider;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 
@@ -25,23 +19,22 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class AddDateFormatsToTemplatesMigration25 extends Migration {
+public class V25AddDateFormatsToTemplatesMigration extends Migration {
 
     private static final String NAME = "Migration for adding date formats files to dossier templates";
     private static final long VERSION = 25;
 
     @Autowired
-    DateFormatsPersistenceService dateFormatsPersistenceService;
+    private DateFormatsPersistenceService dateFormatsPersistenceService;
     @Autowired
-    DossierTemplatePersistenceService dossierTemplatePersistenceService;
+    private DossierTemplatePersistenceService dossierTemplatePersistenceService;
     @Autowired
-    DefaultDateFormatsProvider defaultDateFormatsProvider;
-
-    @Value("${application.type}")
-    private String applicationType;
+    private DefaultDateFormatsProvider defaultDateFormatsProvider;
+    @Autowired
+    private CurrentApplicationTypeProvider currentApplicationTypeProvider;
 
 
-    public AddDateFormatsToTemplatesMigration25() {
+    public V25AddDateFormatsToTemplatesMigration() {
 
         super(NAME, VERSION);
     }
@@ -51,7 +44,7 @@ public class AddDateFormatsToTemplatesMigration25 extends Migration {
     @SneakyThrows
     protected void migrate() {
 
-        if (!applicationType.equalsIgnoreCase("DocuMine")) {
+        if (!currentApplicationTypeProvider.isDocuMine()) {
             log.info("Skipping AddDateFormatsToTemplatesMigration25 as application type is not DocuMine!!!");
             return;
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V26RuleFileUpdateMigration.java

@@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class RuleFileUpdateMigration26 extends Migration {
+public class V26RuleFileUpdateMigration extends Migration {
 
     private final DossierTemplateRepository dossierTemplateRepository;
     private final RulesPersistenceService rulesPersistenceService;
@@ -29,7 +29,7 @@ public class RuleFileUpdateMigration26 extends Migration {
     private static final long VERSION = 26;
 
 
-    public RuleFileUpdateMigration26(DossierTemplateRepository dossierTemplateRepository, RulesPersistenceService rulesPersistenceService) {
+    public V26RuleFileUpdateMigration(DossierTemplateRepository dossierTemplateRepository, RulesPersistenceService rulesPersistenceService) {
 
         super(NAME, VERSION);
         this.dossierTemplateRepository = dossierTemplateRepository;
@@ -62,7 +62,7 @@ public class RuleFileUpdateMigration26 extends Migration {
                     TenantContext.setTenantId(tenantId);
                     String updatedRules = ruleSet.getValue()
                             .replaceAll("import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;",
-                                        "import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;");
+                                        "import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;");
                     rulesPersistenceService.setRules(updatedRules, ruleSet.getDossierTemplateId(), RuleFileType.ENTITY);
                 });
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V27QueueRenameMigration.java

@@ -14,7 +14,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class QueueRenameMigration27 extends Migration {
+public class V27QueueRenameMigration extends Migration {
 
     private final AmqpAdmin amqpAdmin;
 
@@ -86,7 +86,7 @@ public class QueueRenameMigration27 extends Migration {
                                                            "visual_layout_parsing_service_response_queue");
 
 
-    public QueueRenameMigration27(AmqpAdmin amqpAdmin) {
+    public V27QueueRenameMigration(AmqpAdmin amqpAdmin) {
 
         super(NAME, VERSION);
         this.amqpAdmin = amqpAdmin;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V28AddTechnicalNameToJustifications.java

@@ -12,15 +12,15 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Setter
 @Service
-public class AddTechnicalNameToJustifications22 extends Migration {
+public class V28AddTechnicalNameToJustifications extends Migration {
 
     private static final String NAME = "Migration to add a technical name to justifications";
-    private static final long VERSION = 22;
+    private static final long VERSION = 28;
     @Autowired
     private LegalBasisMigrationService legalBasisMigrationService;
 
 
-    public AddTechnicalNameToJustifications22() {
+    public V28AddTechnicalNameToJustifications() {
 
         super(NAME, VERSION);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V29TechnicalNameEntityLogMigration.java

@@ -0,0 +1,151 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.migration.LegalBasisMappingService;
+import com.iqser.red.service.persistence.management.v1.processor.migration.LegalBasisMappingService.TechnicalNameResult;
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class V29TechnicalNameEntityLogMigration extends Migration {
+
+    private final DossierRepository dossierRepository;
+    private final FileRepository fileRepository;
+    private final EntityLogMongoService entityLogMongoService;
+    private final LegalBasisMappingService legalBasisMappingService;
+
+    private static final String NAME = "Migration for technical name update in entity logs";
+    private static final long VERSION = 29;
+
+
+    public V29TechnicalNameEntityLogMigration(DossierRepository dossierRepository,
+                                              FileRepository fileRepository,
+                                              EntityLogMongoService entityLogMongoService,
+                                              LegalBasisMappingService legalBasisMappingService) {
+
+        super(NAME, VERSION);
+        this.dossierRepository = dossierRepository;
+        this.fileRepository = fileRepository;
+        this.entityLogMongoService = entityLogMongoService;
+        this.legalBasisMappingService = legalBasisMappingService;
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        AtomicInteger totalEntriesProcessed = new AtomicInteger(0);
+        AtomicInteger totalValuesHashed = new AtomicInteger(0);
+        AtomicInteger totalValuesUpdatedNormally = new AtomicInteger(0);
+
+        List<DossierEntity> dossiers = dossierRepository.findAll();
+
+        for (DossierEntity dossier : dossiers) {
+            String dossierId = dossier.getId();
+            String dossierTemplateId = dossier.getDossierTemplateId();
+
+
+            Map<String, String> reasonToTechnicalNameMap = legalBasisMappingService.getReasonToTechnicalNameMap(dossierTemplateId);
+
+            List<FileEntity> files = fileRepository.findByDossierId(dossierId);
+
+            for (FileEntity file : files) {
+                String fileId = file.getId();
+
+                Optional<EntityLog> optionalEntityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId);
+
+                if (optionalEntityLog.isPresent()) {
+                    EntityLog entityLog = optionalEntityLog.get();
+
+                    List<EntityLogLegalBasis> legalBasisList = entityLog.getLegalBasis();
+                    if (legalBasisList == null) {
+                        legalBasisList = new java.util.ArrayList<>();
+                        entityLog.setLegalBasis(legalBasisList);
+                    }
+
+                    boolean updated = false;
+
+                    for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
+                        totalEntriesProcessed.getAndIncrement();
+                        String currentLegalBasis = entry.getLegalBasis();
+
+                        if (currentLegalBasis != null && !currentLegalBasis.isEmpty()) {
+                            TechnicalNameResult result = legalBasisMappingService.processLegalBasisAndEntityLogLegalBasisList(currentLegalBasis,
+                                                                                                                                                       reasonToTechnicalNameMap,
+                                                                                                                                                       legalBasisList);
+
+                            if (result != null && !currentLegalBasis.equals(result.technicalName())) {
+                                entry.setLegalBasis(result.technicalName());
+                                updated = true;
+
+                                if (result.hashed()) {
+                                    totalValuesHashed.getAndIncrement();
+                                } else {
+                                    totalValuesUpdatedNormally.getAndIncrement();
+                                }
+                            }
+                        }
+
+                        if (entry.getChanges() != null) {
+                            for (Change change : entry.getChanges()) {
+                                updated |= updatePropertyChanges(change.getPropertyChanges(), reasonToTechnicalNameMap, legalBasisList);
+                            }
+                        }
+
+                        if (entry.getManualChanges() != null) {
+                            for (ManualChange manualChange : entry.getManualChanges()) {
+                                updated |= updatePropertyChanges(manualChange.getPropertyChanges(), reasonToTechnicalNameMap, legalBasisList);
+                            }
+                        }
+                    }
+
+                    if (updated) {
+                        entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog);
+                        log.info("Updated EntityLog for dossierId: {}, fileId: {}", dossierId, fileId);
+                    }
+                }
+            }
+        }
+
+        log.info("Migration completed.");
+        log.info("Total entries processed: {}", totalEntriesProcessed.get());
+        log.info("Total values updated normally: {}", totalValuesUpdatedNormally.get());
+        log.info("Total values hashed: {}", totalValuesHashed.get());
+    }
+
+
+    private boolean updatePropertyChanges(Map<String, String> propertyChanges, Map<String, String> reasonToTechnicalNameMap, List<EntityLogLegalBasis> legalBasisList) {
+
+        boolean updated = false;
+        if (propertyChanges != null && propertyChanges.containsKey("legalBasis")) {
+            String legalBasisValue = propertyChanges.get("legalBasis");
+
+            String updatedLegalBasisValue = legalBasisMappingService.processTextAndEntityLogLegalBasisList(legalBasisValue, reasonToTechnicalNameMap, legalBasisList);
+
+            if (!legalBasisValue.equals(updatedLegalBasisValue)) {
+                propertyChanges.put("legalBasis", updatedLegalBasisValue);
+                updated = true;
+            }
+        }
+        return updated;
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V30TechnicalNameRuleFileMigration.java

@@ -0,0 +1,118 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Setter
+@Service
+public class V30TechnicalNameRuleFileMigration extends Migration {
+
+    private final DossierTemplateRepository dossierTemplateRepository;
+    private final RulesPersistenceService rulesPersistenceService;
+    private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
+
+    private static final String NAME = "Migration for replacing legal basis reasons with technical names in rule files";
+    private static final long VERSION = 30;
+
+
+    public V30TechnicalNameRuleFileMigration(DossierTemplateRepository dossierTemplateRepository,
+                                             RulesPersistenceService rulesPersistenceService,
+                                             LegalBasisMappingPersistenceService legalBasisMappingPersistenceService) {
+
+        super(NAME, VERSION);
+        this.dossierTemplateRepository = dossierTemplateRepository;
+        this.rulesPersistenceService = rulesPersistenceService;
+        this.legalBasisMappingPersistenceService = legalBasisMappingPersistenceService;
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Migration: Updating rule files by replacing legal basis reasons with technical names");
+        updateRuleFiles();
+    }
+
+
+    private void updateRuleFiles() {
+
+        List<DossierTemplateEntity> dossierTemplates = dossierTemplateRepository.findAll();
+        String tenantId = TenantContext.getTenantId();
+
+        dossierTemplates.parallelStream()
+                .forEach(dossierTemplate -> {
+                    TenantContext.setTenantId(tenantId);
+                    String dossierTemplateId = dossierTemplate.getId();
+
+                    List<LegalBasisEntity> legalBasisMappings = legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId);
+
+                    if (legalBasisMappings == null || legalBasisMappings.isEmpty()) {
+                        log.warn("No legal basis mappings found for dossierTemplateId: {}", dossierTemplateId);
+                        return;
+                    }
+
+                    Map<String, String> reasonToTechnicalNameMap = legalBasisMappings.stream()
+                            .filter(lb -> StringUtils.isNotBlank(lb.getReason()) && StringUtils.isNotBlank(lb.getTechnicalName()) && !lb.getReason().equals(lb.getTechnicalName()))
+                            .collect(Collectors.toMap(LegalBasisEntity::getReason, LegalBasisEntity::getTechnicalName, (existing, replacement) -> replacement));
+
+                    if (reasonToTechnicalNameMap.isEmpty()) {
+                        log.warn("No valid mappings to replace for dossierTemplateId: {}", dossierTemplateId);
+                        return;
+                    }
+
+                    Optional<RuleSetEntity> optionalRuleSet = rulesPersistenceService.getRules(dossierTemplateId, RuleFileType.ENTITY);
+
+                    if (optionalRuleSet.isPresent()) {
+                        String originalRulesContent = optionalRuleSet.get().getValue();
+                        String updatedRulesContent = replaceReasonsWithTechnicalNames(originalRulesContent, reasonToTechnicalNameMap);
+
+                        if (!updatedRulesContent.equals(originalRulesContent)) {
+                            rulesPersistenceService.setRules(updatedRulesContent, dossierTemplateId, RuleFileType.ENTITY);
+                            log.info("Updated rule file for dossierTemplateId: {}", dossierTemplateId);
+                        } else {
+                            log.info("No replacements made for dossierTemplateId: {}", dossierTemplateId);
+                        }
+                    } else {
+                        log.warn("No rule set found for dossierTemplateId: {}", dossierTemplateId);
+                    }
+                });
+    }
+
+
+    private String replaceReasonsWithTechnicalNames(String rulesContent, Map<String, String> reasonToTechnicalNameMap) {
+
+        String rulesContentResult = rulesContent;
+        for (Map.Entry<String, String> entry : reasonToTechnicalNameMap.entrySet()) {
+            String reason = entry.getKey();
+            String technicalName = entry.getValue();
+            rulesContentResult = StringUtils.replace(rulesContentResult, quoteString(reason), quoteString(technicalName));
+        }
+        return rulesContentResult;
+    }
+
+
+    private static String quoteString(String string) {
+
+        return "\"" + string + "\"";
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V31TechnicalNameImportedFilesMigration.java

@@ -0,0 +1,128 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.migration.LegalBasisMappingService;
+import com.iqser.red.service.persistence.management.v1.processor.migration.LegalBasisMappingService.TechnicalNameResult;
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class V31TechnicalNameImportedFilesMigration extends Migration {
+
+    private static final String NAME = "Migration for replacing legal basis reasons with technical names in imported redactions and legal bases files";
+    private static final long VERSION = 31;
+
+    private final DossierRepository dossierRepository;
+    private final FileRepository fileRepository;
+    private final FileManagementStorageService fileManagementStorageService;
+    private final LegalBasisMappingService legalBasisMappingService;
+
+
+    public V31TechnicalNameImportedFilesMigration(DossierRepository dossierRepository,
+                                                  FileRepository fileRepository,
+                                                  FileManagementStorageService fileManagementStorageService,
+                                                  LegalBasisMappingService legalBasisMappingService) {
+
+        super(NAME, VERSION);
+        this.dossierRepository = dossierRepository;
+        this.fileRepository = fileRepository;
+        this.fileManagementStorageService = fileManagementStorageService;
+        this.legalBasisMappingService = legalBasisMappingService;
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Migration: Updating imported redactions and legal bases files by replacing legal basis reasons with technical names");
+
+        List<DossierEntity> dossiers = dossierRepository.findAll();
+
+        for (DossierEntity dossier : dossiers) {
+            String dossierId = dossier.getId();
+            String dossierTemplateId = dossier.getDossierTemplateId();
+
+            Map<String, String> reasonToTechnicalNameMap = legalBasisMappingService.getReasonToTechnicalNameMap(dossierTemplateId);
+
+            List<FileEntity> files = fileRepository.findByDossierId(dossierId);
+
+            for (FileEntity file : files) {
+                String fileId = file.getId();
+
+                if (fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_REDACTIONS)) {
+                    ImportedRedactionsPerPage importedRedactions = fileManagementStorageService.getImportedRedactions(dossierId, fileId);
+
+                    boolean updated = false;
+
+                    for (ImportedRedaction redaction : importedRedactions.getImportedRedactions().values().stream().flatMap(Collection::stream).toList()) {
+                        String currentLegalBasis = redaction.getLegalBasis();
+
+                        if (currentLegalBasis != null && !currentLegalBasis.isEmpty()) {
+                            TechnicalNameResult result = legalBasisMappingService.processLegalBasis(currentLegalBasis, reasonToTechnicalNameMap);
+
+                            if (result != null && !currentLegalBasis.equals(result.technicalName())) {
+                                redaction.setLegalBasis(result.technicalName());
+                                updated = true;
+                            }
+                        }
+
+                    }
+
+                    if (updated) {
+                        fileManagementStorageService.storeJSONObject(dossierId, fileId, FileType.IMPORTED_REDACTIONS, importedRedactions);
+                        log.info("Updated imported redactions for dossierId: {}, fileId: {}", dossierId, fileId);
+                    }
+                }
+
+                if (fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES)) {
+                    ImportedLegalBases importedLegalBases = fileManagementStorageService.getImportedLegalBases(dossierId, fileId);
+
+                    boolean updated = false;
+
+                    List<ImportedLegalBasis> legalBases = importedLegalBases.getImportedLegalBases();
+
+                    for (ImportedLegalBasis importedLegalBasis : legalBases) {
+                        String currentReason = importedLegalBasis.getReason();
+                        String currentTechnicalName = importedLegalBasis.getTechnicalName();
+
+                        if (currentTechnicalName == null || currentTechnicalName.isEmpty()) {
+                            if (currentReason != null && !currentReason.isEmpty()) {
+                                TechnicalNameResult result = legalBasisMappingService.processLegalBasis(currentReason, reasonToTechnicalNameMap);
+
+                                if (result != null) {
+                                    importedLegalBasis.setTechnicalName(result.technicalName());
+                                    updated = true;
+                                }
+                            }
+                        }
+                    }
+
+                    if (updated) {
+                        fileManagementStorageService.storeJSONObject(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES, importedLegalBases);
+                        log.info("Updated imported legal bases for dossierId: {}, fileId: {}", dossierId, fileId);
+                    }
+                }
+            }
+        }
+
+        log.info("Migration completed.");
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/V32RankDeduplicationMigration.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class V32RankDeduplicationMigration extends Migration {
+
+    private static final String NAME = "Adding to the migration the rank de-duplication";
+    private static final long VERSION = 32;
+
+    private final RankDeDuplicationService rankDeDuplicationService;
+
+
+    public V32RankDeduplicationMigration(RankDeDuplicationService rankDeDuplicationService) {
+
+        super(NAME, VERSION);
+        this.rankDeDuplicationService = rankDeDuplicationService;
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        log.info("Migration: Checking for duplicate ranks");
+        rankDeDuplicationService.deduplicate();
+
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/AnalysisType.java

@@ -4,6 +4,7 @@ public enum AnalysisType {
 
     DEFAULT,
     MANUAL_REDACTION_REANALYZE,
+    FORCE_ANALYSE,
     COMPONENTS_ONLY_REANALYZE
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/DossierIdFilterable.java

@@ -0,0 +1,6 @@
+package com.iqser.red.service.persistence.management.v1.processor.model;
+
+public interface DossierIdFilterable {
+
+    String getDossierId();
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java

@@ -168,6 +168,10 @@ public final class ActionRoles {
     public static final String READ_APP_CONFIG = "red-read-app-configuration";
     public static final String WRITE_APP_CONFIG = "red-write-app-configuration";
 
+    // USER STATS
+
+    public static final String READ_USER_STATS = "red-get-user-stats";
+
     // License Management
     public static final String UPDATE_LICENSE = "red-update-license";
     public static final String READ_LICENSE = "red-read-license";

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ApplicationRoles.java

@@ -25,8 +25,7 @@ public final class ApplicationRoles {
                                                                                   UPDATE_LICENSE,
                                                                                   GET_TENANTS,
                                                                                   CREATE_TENANT,
-                                                                                  READ_USERS,
-                                                                                  READ_ALL_USERS,
+                                                                                  READ_USERS, READ_ALL_USERS, READ_USER_STATS,
                                                                                   WRITE_USERS,
                                                                                   READ_SMTP_CONFIGURATION,
                                                                                   WRITE_SMTP_CONFIGURATION,
@@ -62,8 +61,7 @@ public final class ApplicationRoles {
                                                                             PROCESS_MANUAL_REDACTION_REQUEST,
                                                                             READ_COLORS,
                                                                             READ_DICTIONARY_TYPES,
-                                                                            READ_DIGITAL_SIGNATURE,
-                                                                            READ_DOSSIER,
+                                                                            READ_DIGITAL_SIGNATURE, READ_DOSSIER,
                                                                             READ_DOSSIER_ATTRIBUTES,
                                                                             READ_DOSSIER_ATTRIBUTES_CONFIG,
                                                                             READ_DOSSIER_TEMPLATES,
@@ -118,8 +116,7 @@ public final class ApplicationRoles {
                                                                              READ_DOSSIER_TEMPLATES,
                                                                              READ_FILE_ATTRIBUTES_CONFIG,
                                                                              READ_LEGAL_BASIS,
-                                                                             READ_LICENSE_REPORT,
-                                                                             READ_NOTIFICATIONS,
+                                                                             READ_LICENSE_REPORT, READ_NOTIFICATIONS, READ_USER_STATS,
                                                                              READ_RULES,
                                                                              READ_DATA_FORMATS,
                                                                              READ_SMTP_CONFIGURATION,
@@ -153,9 +150,8 @@ public final class ApplicationRoles {
                                                                                   READ_APP_CONFIG,
                                                                                   READ_GENERAL_CONFIGURATION,
                                                                                   READ_GENERAL_CONFIGURATION,
-                                                                                  GET_SIMILAR_IMAGES,
-                                                                                  READ_NOTIFICATIONS,
-                                                                                  READ_USERS,
+                                                                                  GET_SIMILAR_IMAGES, READ_NOTIFICATIONS,
+                                                                                  READ_USERS, READ_USER_STATS,
                                                                                   UPDATE_MY_PROFILE,
                                                                                   UPDATE_NOTIFICATIONS,
                                                                                   WRITE_USERS,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AccessControlService.java

@@ -1,6 +1,9 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
+import java.util.Set;
+
 import org.springframework.http.HttpStatus;
+import org.springframework.security.access.AccessDeniedException;
 import org.springframework.security.access.prepost.PostAuthorize;
 import org.springframework.security.acls.AclPermissionEvaluator;
 import org.springframework.security.core.context.SecurityContextHolder;
@@ -203,4 +206,19 @@ public class AccessControlService {
         }
     }
 
+    public void verifyUserIsDossierOwnerOrApproverOrAssignedReviewer(String dossierId, Set<String> fileIds) {
+
+        try {
+            verifyUserIsDossierOwnerOrApprover(dossierId);
+        } catch (AccessDeniedException e1) {
+            try {
+                for (String fileId : fileIds) {
+                    verifyUserIsReviewer(dossierId, fileId);
+                }
+            } catch (NotAllowedException e2) {
+                throw new NotAllowedException("User must be dossier owner, approver or assigned reviewer of the file.");
+            }
+        }
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ApprovalVerificationService.java

@@ -51,7 +51,7 @@ public class ApprovalVerificationService {
                     addWarning(entry, WarningType.LEGAL_BASIS_MISSING, approveResponse);
                 } else {
                     var legalBasisEntity = legalBasisMappings.stream()
-                            .filter(mapping -> mapping.getReason().equals(entry.getLegalBasis()))
+                            .filter(mapping -> mapping.getTechnicalName().equals(entry.getLegalBasis()))
                             .findFirst();
                     if (legalBasisEntity.isEmpty() || StringUtils.isEmpty(legalBasisEntity.get().getTechnicalName())) {
                         addWarning(entry, WarningType.UNMAPPED_JUSTIFICATION, approveResponse);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java

@@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.management.v1.processor.mapper.Componen
 import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping;
 import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
 import com.opencsv.CSVParserBuilder;
 import com.opencsv.CSVReader;
@@ -107,7 +108,7 @@ public class ComponentMappingService {
                                                     String fileName,
                                                     char quoteChar) {
 
-        Charset charset = resolveCharset(encoding);
+        Charset charset = StringEncodingUtils.resolveCharset(encoding);
 
         CsvStats stats = sortCSVFile(delimiter, mappingFile, charset, quoteChar);
 
@@ -126,20 +127,6 @@ public class ComponentMappingService {
     }
 
 
-    private static Charset resolveCharset(String encoding) {
-
-        try {
-            return Charset.forName(encoding);
-        } catch (IllegalCharsetNameException e) {
-            throw new BadRequestException("Invalid character encoding: " + encoding);
-        } catch (UnsupportedCharsetException e) {
-            throw new BadRequestException("Unsupported character encoding: " + encoding);
-        } catch (IllegalArgumentException e) {
-            throw new BadRequestException("Encoding can't be null.");
-        }
-    }
-
-
     private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset, char quoteChar) throws BadRequestException, IOException {
 
         Path tempFile = Files.createTempFile("mapping", ".tmp");

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CurrentApplicationTypeProvider.java

@@ -0,0 +1,44 @@
+package com.iqser.red.service.persistence.management.v1.processor.service;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.tenantcommons.TenantApplicationType;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+import com.knecon.fforesight.tenantcommons.TenantProvider;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class CurrentApplicationTypeProvider {
+
+    TenantProvider tenantProvider;
+
+    Map<String, TenantApplicationType> cache = new ConcurrentHashMap<>();
+
+
+    public TenantApplicationType get() {
+
+        return cache.computeIfAbsent(TenantContext.getTenantId(), tenantProvider::getTenantApplicationType);
+    }
+
+
+    public boolean isDocuMine() {
+
+        return get().equals(TenantApplicationType.DocuMine);
+    }
+
+
+    public boolean isRedactManager() {
+
+        return get().equals(TenantApplicationType.RedactManager);
+    }
+
+}
+

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java

@@ -100,6 +100,8 @@ public class DictionaryManagementService {
                                                                            typeRequest.isCaseInsensitive(),
                                                                            typeRequest.isRecommendation(),
                                                                            typeRequest.getDescription(),
+                                                                           typeRequest.isAiCreationEnabled(),
+                                                                           typeRequest.getAiDescription(),
                                                                            typeRequest.isAddToDictionaryAction(),
                                                                            typeRequest.getLabel(),
                                                                            typeRequest.getDossierId(),
@@ -237,10 +239,11 @@ public class DictionaryManagementService {
 
     @Transactional
     public void addEntries(String typeId, List<String> entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType) {
+
         addEntries(typeId, entries, removeCurrent, ignoreInvalidEntries, dictionaryEntryType, false);
     }
-    
-    
+
+
     @Transactional
     public void addEntries(String typeId, List<String> entries, boolean removeCurrent, boolean ignoreInvalidEntries, DictionaryEntryType dictionaryEntryType, boolean isImport) {
 
@@ -291,7 +294,10 @@ public class DictionaryManagementService {
         // check for the existence of dossier type and create in case it does not exist
         if (isDossierTypeId(typeId)) {
             try {
-                dictionaryPersistenceService.getType(typeId);
+                TypeEntity type = dictionaryPersistenceService.getType(typeId, true);
+                if (type.isDeleted()) {
+                    dictionaryPersistenceService.undeleteType(typeId);
+                }
             } catch (NotFoundException e) {
                 // type not found check first dossier is matching the specified dossier template
                 var dossierId = getDossierIdFromTypeId(typeId);
@@ -395,9 +401,13 @@ public class DictionaryManagementService {
 
         dictionaryPersistenceService.deleteType(typeId);
 
-        entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
-        entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
-        entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
+        if (isDossierTypeId(typeId)) {
+            entryPersistenceService.hardDeleteAllEntriesForTypeId(typeId);
+        } else {
+            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
+            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
+            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
+        }
 
         dictionaryPersistenceService.incrementVersion(typeId);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java

@@ -178,6 +178,8 @@ public class DictionaryService {
                                                             .isCaseInsensitive(typeValue.isCaseInsensitive())
                                                             .isRecommendation(typeValue.isRecommendation())
                                                             .description(typeValue.getDescription())
+                                                            .aiCreationEnabled(typeValue.isAiCreationEnabled())
+                                                            .aiDescription(typeValue.getAiDescription())
                                                             .addToDictionaryAction(typeValue.isAddToDictionaryAction())
                                                             .label(typeValue.getLabel())
                                                             .hasDictionary(typeValue.isHasDictionary())
@@ -201,6 +203,8 @@ public class DictionaryService {
                                                                 .isCaseInsensitive(typeValue.isCaseInsensitive())
                                                                 .isRecommendation(typeValue.isRecommendation())
                                                                 .description(typeValue.getDescription())
+                                                                .aiCreationEnabled(typeValue.isAiCreationEnabled())
+                                                                .aiDescription(typeValue.getAiDescription())
                                                                 .addToDictionaryAction(typeEntity.isAddToDictionaryAction())
                                                                 .label(typeValue.getLabel())
                                                                 .hasDictionary(typeValue.isHasDictionary())
@@ -233,6 +237,8 @@ public class DictionaryService {
                                                            .isCaseInsensitive(typeValue.isCaseInsensitive())
                                                            .isRecommendation(typeValue.isRecommendation())
                                                            .description(typeValue.getDescription())
+                                                           .aiCreationEnabled(typeValue.isAiCreationEnabled())
+                                                           .aiDescription(typeValue.getAiDescription())
                                                            .addToDictionaryAction(typeValue.isAddToDictionaryAction())
                                                            .label(typeValue.getLabel())
                                                            .hasDictionary(typeValue.isHasDictionary())
@@ -262,7 +268,7 @@ public class DictionaryService {
     @PreAuthorize("hasAuthority('" + DELETE_DOSSIER_DICTIONARY_TYPE + "')")
     public void deleteDossierType(String type, String dossierTemplateId, String dossierId) {
 
-        accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
+        accessControlService.checkAccessPermissionsToDossier(dossierId);
         accessControlService.verifyUserIsMemberOrApprover(dossierId);
         deleteType(toTypeId(type, dossierTemplateId, dossierId));
     }
@@ -297,6 +303,8 @@ public class DictionaryService {
                         .caseInsensitive(typeResult.isCaseInsensitive())
                         .recommendation(typeResult.isRecommendation())
                         .description(typeResult.getDescription())
+                        .aiCreationEnabled(typeResult.isAiCreationEnabled())
+                        .aiDescription(typeResult.getAiDescription())
                         .addToDictionaryAction(typeResult.isAddToDictionaryAction())
                         .label(typeResult.getLabel())
                         .hasDictionary(typeResult.isHasDictionary())
@@ -486,13 +494,9 @@ public class DictionaryService {
     }
 
     @PreAuthorize("hasAuthority('" + ADD_UPDATE_DICTIONARY_TYPE + "')")
-    public void changeAddToDictionary(String type, String dossierTemplateId, String dossierId, boolean addToDictionary) {
+    public void changeAddToDictionary(String type, String dossierTemplateId, String dossierId, boolean addToDictionaryAction) {
 
-        var typeEntity = dictionaryPersistenceService.getType(toTypeId(type, dossierTemplateId, dossierId));
-        if (typeEntity.isDossierDictionaryOnly()) {
-            typeEntity.setAddToDictionaryAction(addToDictionary);
-            dictionaryPersistenceService.saveType(typeEntity);
-        }
+        dictionaryPersistenceService.updateAddToDictionary(toTypeId(type, dossierTemplateId, dossierId), addToDictionaryAction);
 
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.utils.DossierMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierInformation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
@@ -275,4 +276,16 @@ public class DossierManagementService {
 
     }
 
+
+    public DossierChangeResponseV2 changesSinceV2(JSONPrimitive<OffsetDateTime> since) {
+        return dossierService.changesSinceV2(since.getValue());
+    }
+
+
+    @Transactional
+    public List<Dossier> getDossiersByIds(Set<String> viewableDossierIds) {
+
+        return getConvertedAllDossiers(dossierService.getAllDossiers(viewableDossierIds), true,true);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
 import java.time.OffsetDateTime;
+import java.util.Collection;
 import java.util.List;
 import java.util.Set;
 
@@ -14,6 +15,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierChangeResponseV2;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierChange;
@@ -147,7 +149,7 @@ public class DossierService {
     }
 
 
-    public List<DossierEntity> getAllDossiers(List<String> dossierIds) {
+    public List<DossierEntity> getAllDossiers(Collection<String> dossierIds) {
 
         return dossierPersistenceService.findAllDossiers(dossierIds);
     }
@@ -188,5 +190,8 @@ public class DossierService {
     }
 
 
+    public DossierChangeResponseV2 changesSinceV2(OffsetDateTime value) {
+        return dossierPersistenceService.hasChangesSinceV2(value);
+    }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java

@@ -156,7 +156,7 @@ public class DossierTemplateCloneService {
 
     private void cloneComponents(String dossierTemplate, String clonedDossierTemplateId) {
 
-        List<ComponentDefinitionEntity> componentDefinitionEntities = componentDefinitionPersistenceService.findComponentsByDossierTemplateId(dossierTemplate);
+        List<ComponentDefinitionEntity> componentDefinitionEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierTemplate);
 
         for (ComponentDefinitionEntity componentDefinitionEntity : componentDefinitionEntities) {
             ComponentDefinitionAddRequest componentDefinitionAddRequest = ComponentDefinitionAddRequest.builder()
@@ -237,6 +237,8 @@ public class DossierTemplateCloneService {
                                                             t.isCaseInsensitive(),
                                                             t.isRecommendation(),
                                                             t.getDescription(),
+                                                            t.isAiCreationEnabled(),
+                                                            t.getAiDescription(),
                                                             t.isAddToDictionaryAction(),
                                                             t.getLabel(),
                                                             null,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java

@@ -103,13 +103,17 @@ public class EntityLogMergeService {
                                        int analysisNumber,
                                        Map<String, List<BaseAnnotation>> allManualChanges) {
 
-        Map<String, String> trackLocalChangesBasedOnDictEntriesMap = new HashMap<>();
         List<EntityLogEntry> mergedEntityLogEntries = new LinkedList<>(entityLogEntries);
         Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber)//
                 .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
         mergedEntityLogEntries.addAll(addedLocalManualEntries.values());
         buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(mergedEntityLogEntries::add);
 
+        Map<String, String> trackLocalChangesBasedOnDictEntriesMap = unprocessedManualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(ManualRedactionEntry::isLocal)
+                .filter(entry -> entry.getSourceId() != null && !entry.getSourceId().isEmpty())
+                .collect(Collectors.toMap(ManualRedactionEntry::getAnnotationId, ManualRedactionEntry::getSourceId));
         processEntityLogEntries(dossier, mergedEntityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges, trackLocalChangesBasedOnDictEntriesMap);
 
         adjustEntityLogEntriesAfterLocalChangesBasedOnDict(entityLogEntries, trackLocalChangesBasedOnDictEntriesMap, analysisNumber);
@@ -121,7 +125,7 @@ public class EntityLogMergeService {
                                                                     Map<String, String> trackLocalChangesBasedOnDictEntriesMap,
                                                                     int analysisNumber) {
 
-        var dictEntryIdsToUpdate = trackLocalChangesBasedOnDictEntriesMap.values();
+        Set<String> dictEntryIdsToUpdate = new HashSet<>(trackLocalChangesBasedOnDictEntriesMap.values());
         entityLogEntries.stream()
                 .filter(entityLogEntry -> dictEntryIdsToUpdate.contains(entityLogEntry.getId()))
                 .forEach(entityLogEntry -> {
@@ -268,26 +272,14 @@ public class EntityLogMergeService {
                         return null;
                     } else if (localChange instanceof ManualResizeRedaction manualResizeRedaction) {
                         mergeResizeRedaction(manualResizeRedaction, entityLogEntry, analysisNumber);
-                        if (manualResizeRedaction.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualResizeRedaction.getAnnotationId(), manualResizeRedaction.getBasedOnDictAnnotationId());
-                        }
 
                         return null;
                     } else if (localChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
                         mergeLegalBasisChange(manualLegalBasisChange, entityLogEntry, analysisNumber);
-                        if (manualLegalBasisChange.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualLegalBasisChange.getAnnotationId(), manualLegalBasisChange.getBasedOnDictAnnotationId());
-                        }
                         return null;
                     } else if (localChange instanceof ManualRecategorization manualRecategorization) {
-                        if (manualRecategorization.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualRecategorization.getAnnotationId(), manualRecategorization.getBasedOnDictAnnotationId());
-                        }
                         return mergeRecategorization(manualRecategorization, entityLogEntry, dossier, analysisNumber);
                     } else if (localChange instanceof ManualForceRedaction manualForceRedaction) {
-                        if (manualForceRedaction.getBasedOnDictAnnotationId() != null) {
-                            trackLocalChangesBasedOnDictEntriesMap.put(manualForceRedaction.getAnnotationId(), manualForceRedaction.getBasedOnDictAnnotationId());
-                        }
                         mergeForceRedaction(manualForceRedaction, entityLogEntry, analysisNumber);
                         return null;
                     } else {
@@ -397,7 +389,6 @@ public class EntityLogMergeService {
             entityLogEntry.setState(EntryState.REMOVED);
             change.setType(ChangeType.REMOVED);
         }
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(idRemoval, 0));
 
         changes.add(change);
@@ -435,7 +426,6 @@ public class EntityLogMergeService {
         entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
         entityLogEntry.setPositions(newPositions);
         entityLogEntry.setValue(manualResizeRedaction.getValue());
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualResizeRedaction, 0));
     }
 
@@ -464,7 +454,6 @@ public class EntityLogMergeService {
         entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
         entityLogEntry.setSection(manualLegalBasisChange.getSection());
         entityLogEntry.setValue(manualLegalBasisChange.getValue());
-        entityLogEntry.getEngines().add(Engine.MANUAL);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualLegalBasisChange, 0));
     }
 
@@ -486,8 +475,6 @@ public class EntityLogMergeService {
             return pendingEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
         }
 
-        entityLogEntry.getEngines().add(Engine.MANUAL);
-
         if (recategorization.getType() != null && !recategorization.getType().equals(entityLogEntry.getType())) {
             boolean isHint = isHint(recategorization.getType(), dossier);
             entityLogEntry.setType(recategorization.getType());
@@ -549,10 +536,6 @@ public class EntityLogMergeService {
                                            PropertyChange.builder().property("state").oldValue(oldState.name()).newValue(newState.name()).build()));
         entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
         entityLogEntry.setState(newState);
-        entityLogEntry.getEngines().add(Engine.MANUAL);
-        if (forceRedaction.getBasedOnDictAnnotationId() != null) {
-            entityLogEntry.getEngines().add(Engine.DICTIONARY);
-        }
         addChanges(entityLogEntry, changes);
         entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(forceRedaction, 0));
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
+import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ASCII_ENCODING;
+import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ISO_ENCODING;
 import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeTypeFormats.FILE_ATTRIBUTE_TYPE_DATE_FORMAT;
 import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeTypeFormats.FILE_ATTRIBUTE_TYPE_NUMBER_REGEX;
 
@@ -32,6 +34,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Confl
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ImportCsvRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ImportCsvResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
@@ -59,10 +62,6 @@ public class FileAttributesManagementService {
     private final DossierPersistenceService dossierPersistenceService;
     private final IndexingService indexingService;
 
-    public static String UTF_ENCODING = "UTF-8";
-    public static String ASCII_ENCODING = "ASCII";
-    public static String ISO_ENCODING = "ISO";
-
 
     @Transactional
     public ImportCsvResponse importCsv(String dossierId, ImportCsvRequest importCsvRequest) {
@@ -144,7 +143,7 @@ public class FileAttributesManagementService {
             throw new IllegalArgumentException("Delimiter must be a single character.");
         }
         char delimiterChar = delimiter.charAt(0);
-        Charset charset = Charset.forName(encoding);
+        Charset charset = StringEncodingUtils.resolveCharset(encoding);
 
         try (CSVReader csvReader = new CSVReaderBuilder(new InputStreamReader(new ByteArrayInputStream(csvFileBytes), charset)).withCSVParser(new CSVParserBuilder().withSeparator(
                 delimiterChar).build()).build()) {
@@ -214,7 +213,7 @@ public class FileAttributesManagementService {
         if (ASCII_ENCODING.equalsIgnoreCase(encoding) || StandardCharsets.US_ASCII.name().equalsIgnoreCase(encoding)) {
             return StandardCharsets.US_ASCII;
         }
-        // accept both "ISO" (non-unique name) and the actual name "US-ASCII" of the charset
+        // accept only name "ISO_8859_1" of the charset
         if (ISO_ENCODING.equalsIgnoreCase(encoding) || StandardCharsets.ISO_8859_1.name().equalsIgnoreCase(encoding)) {
             return StandardCharsets.ISO_8859_1;
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java

@@ -20,7 +20,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
 import com.iqser.red.service.search.v1.model.IndexMessageType;
 
-import groovy.transform.Field;
 import jakarta.transaction.Transactional;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -44,7 +43,6 @@ public class FileDeletionService {
     FileStatusPersistenceService fileStatusPersistenceService;
     FileManagementStorageService fileManagementStorageService;
     IndexingService indexingService;
-    ComponentLogService componentLogService;
     ComponentLogMongoService componentLogMongoService;
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
-import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
 
 import java.io.BufferedInputStream;
 import java.io.File;
@@ -13,20 +12,20 @@ import java.util.List;
 import org.apache.commons.io.IOUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
-import com.iqser.red.storage.commons.exception.StorageException;
+import com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto;
+import com.iqser.red.service.redaction.v1.server.data.DocumentStructureWrapper;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
@@ -170,6 +169,7 @@ public class FileManagementStorageService {
         return entityLogMongoService.entityLogDocumentExists(dossierId, fileId);
     }
 
+
     public boolean componentLogExists(String dossierId, String fileId) {
 
         return componentLogMongoService.componentLogDocumentExists(dossierId, fileId);
@@ -189,12 +189,12 @@ public class FileManagementStorageService {
     }
 
 
-    public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
+    public ImportedRedactionsPerPage getImportedRedactions(String dossierId, String fileId) {
 
         try {
             return storageService.readJSONObject(TenantContext.getTenantId(),
                                                  StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
-                                                 ImportedRedactions.class);
+                                                 ImportedRedactionsPerPage.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("ImportedRedactions does not exist");
         } catch (Exception e) {
@@ -202,11 +202,41 @@ public class FileManagementStorageService {
         }
     }
 
+
+    public ImportedLegalBases getImportedLegalBases(String dossierId, String fileId) {
+
+        try {
+            return storageService.readJSONObject(TenantContext.getTenantId(),
+                                                 StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES),
+                                                 ImportedLegalBases.class);
+        } catch (StorageObjectDoesNotExist e) {
+            throw new NotFoundException("ImportedLegalBases does not exist");
+        } catch (Exception e) {
+            throw new RuntimeException("Could not convert ImportedLegalBases", e);
+        }
+    }
+
+
+    public com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions getOldImportedRedactions(String dossierId, String fileId) {
+
+        try {
+            return storageService.readJSONObject(TenantContext.getTenantId(),
+                                                 StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
+                                                 com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions.class);
+        } catch (StorageObjectDoesNotExist e) {
+            throw new NotFoundException("ImportedRedactions does not exist");
+        } catch (Exception e) {
+            throw new RuntimeException("Could not convert ImportedRedactions", e);
+        }
+    }
+
+
     public boolean objectExists(String dossierId, String fileId, String fileName, String fileExtension) {
 
         return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileName, fileExtension));
     }
 
+
     public boolean objectExists(String dossierId, String fileId, FileType origin) {
 
         return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, origin));
@@ -224,11 +254,13 @@ public class FileManagementStorageService {
         storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType));
     }
 
+
     public void deleteObject(String dossierId, String fileId, String fileName, String fileExtension) {
 
         storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileName, fileExtension));
     }
 
+
     public void deleteDocumentAndNerObjects(String dossierId, String fileId) {
 
         deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
@@ -249,8 +281,11 @@ public class FileManagementStorageService {
 
     }
 
+
     public void deleteAllObjects(String dossierId, String fileId) {
 
+        deleteObject(dossierId, fileId, FileType.VIEWER_DOCUMENT);
+
         deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
         deleteEntityLog(dossierId, fileId);
 
@@ -268,6 +303,7 @@ public class FileManagementStorageService {
         entityLogMongoService.deleteEntityLog(dossierId, fileId);
     }
 
+
     public void deleteComponentLog(String dossierId, String fileId) {
 
         componentLogMongoService.deleteComponentLog(dossierId, fileId);
@@ -284,7 +320,7 @@ public class FileManagementStorageService {
 
         return new DocumentStructureWrapper(storageService.readProtoObject(TenantContext.getTenantId(),
                                                                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
-                                                                           DocumentStructure.parser()));
+                                                                           DocumentStructureProto.DocumentStructure.parser()));
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusManagementService.java

@@ -52,6 +52,11 @@ public class FileStatusManagementService {
                 .collect(Collectors.toList());
     }
 
+    public List<FileModel> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) {
+
+        return fileStatusService.findAllDossierIdAndIds(dossierId,fileIds);
+    }
+
 
     public List<String> getDossierStatusIds(String dossierId, boolean includeDeleted) {
 
@@ -120,7 +125,7 @@ public class FileStatusManagementService {
         if (userId != null) {
             assignee = userId;
         }
-        fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_REVIEW : WorkflowStatus.NEW);
+        fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_REVIEW : WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
         fileStatusService.setAssignee(fileId, assignee);
         indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, fileId, 2);
     }
@@ -134,7 +139,7 @@ public class FileStatusManagementService {
             assignee = approverId;
 
         }
-        fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_APPROVAL : WorkflowStatus.NEW);
+        fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_APPROVAL : WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
         fileStatusService.setAssignee(fileId, approverId);
         indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, fileId, 2);
     }
@@ -164,7 +169,7 @@ public class FileStatusManagementService {
             throw new BadRequestException("Allowed transition not possible from: " + fileStatus.getWorkflowStatus() + " to status NEW");
         }
         fileStatusService.setAssignee(fileId, null);
-        fileStatusService.setStatusSuccessful(fileId, WorkflowStatus.NEW);
+        fileStatusService.setStatusSuccessful(fileId, WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java

@@ -43,12 +43,14 @@ public class FileStatusMapper {
                 .hasAnnotationComments(status.isHasAnnotationComments())
                 .uploader(status.getUploader())
                 .dictionaryVersion(status.getDictionaryVersion())
+                .aiCreationVersion(status.getAiCreationVersion())
                 .rulesVersion(status.getRulesVersion())
                 .componentRulesVersion(status.getComponentRulesVersion())
                 .dateFormatsVersion(status.getDateFormatsVersion())
                 .legalBasisVersion(status.getLegalBasisVersion())
                 .lastProcessed(status.getLastProcessed())
                 .lastLayoutProcessed(status.getLastLayoutProcessed())
+                .layoutParserVersion(status.getLayoutParserVersion())
                 .approvalDate(status.getApprovalDate())
                 .lastUploaded(status.getLastUploaded())
                 .analysisDuration(status.getAnalysisDuration())
@@ -68,6 +70,7 @@ public class FileStatusMapper {
                 .fileSize(status.getFileSize())
                 .fileErrorInfo(status.getFileErrorInfo())
                 .componentMappingVersions(status.getComponentMappingVersions())
+                .lastDownload(status.getLastDownloadDate())
                 .build();
 
     }