RED-8762: added endpoint to update flag addToDictionary for given type, dossierTemplateId and dossierId

removed 3.6.x temporary unique constraints

See merge request redactmanager/persistence-service!440

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -6,7 +6,7 @@ plugins {
     jacoco
 }
 
-val redactionServiceVersion by rootProject.extra { "4.199.0" }
+val redactionServiceVersion by rootProject.extra { "4.244.38" }
 val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" }
 val redactionReportServiceVersion by rootProject.extra { "4.47.0" }
 val searchServiceVersion by rootProject.extra { "2.71.0" }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DictionaryController.java

@@ -23,12 +23,10 @@ import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
-import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
-import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
-import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils;
 import com.iqser.red.service.persistence.management.v1.processor.utils.TypeValueMapper;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.DictionaryResource;
@@ -42,6 +40,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import feign.FeignException;
 import jakarta.validation.Valid;
@@ -318,4 +318,14 @@ public class DictionaryController implements DictionaryResource {
                 .build());
     }
 
+
+    @Override
+    public void changeFlags(@PathVariable(TYPE_PARAMETER_NAME) String type,
+                            @PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
+                            @RequestParam(value = DOSSIER_ID_PARAMETER_NAME) String dossierId,
+                            @RequestParam(value = "addToDictionary") boolean addToDictionary) {
+
+        dictionaryService.changeAddToDictionary(type, dossierTemplateId, dossierId, addToDictionary);
+    }
+
 }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java

@@ -16,11 +16,11 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
 import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
-import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionMapper;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionUndoService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
@@ -59,15 +59,14 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ManualRedactionController implements ManualRedactionResource {
 
-    final static String FILE_ID = "fileId";
+    static final String FILE_ID = "fileId";
-    final static String DOSSIER_ID = "dossierId";
+    static final String DOSSIER_ID = "dossierId";
-    final static String ANNOTATION_ID = "annotationId";
+    static final String ANNOTATION_ID = "annotationId";
     ManualRedactionService manualRedactionService;
     ManualRedactionUndoService manualRedactionUndoService;
     DossierManagementService dossierManagementService;
     AuditPersistenceService auditPersistenceService;
     AccessControlService accessControlService;
-    ManualRedactionMapper manualRedactionMapper;
     CommentService commentService;
     FileStatusManagementService fileStatusManagementService;
 
@@ -99,12 +98,12 @@ public class ManualRedactionController implements ManualRedactionResource {
         accessControlService.verifyUserIsReviewerOrApprover(dossierId, fileId);
 
         auditPersistenceService.audit(AuditRequest.builder()
-                .userId(KeycloakSecurity.getUserId())
+                                              .userId(KeycloakSecurity.getUserId())
-                .objectId(fileId)
+                                              .objectId(fileId)
-                .category(AuditCategory.DOCUMENT.name())
+                                              .category(AuditCategory.DOCUMENT.name())
-                .message("Comment was removed.")
+                                              .message("Comment was removed.")
-                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
+                                              .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
-                .build());
+                                              .build());
         commentService.deleteComment(fileId, List.of(Long.valueOf(commentId)));
 
     }
@@ -114,11 +113,13 @@ public class ManualRedactionController implements ManualRedactionResource {
     @PreAuthorize("hasAuthority('" + READ_MANUAL_REDACTIONS + "')")
     public ManualRedactions getManualRedactions(@PathVariable(DOSSIER_ID) String dossierId,
                                                 @PathVariable(FILE_ID) String fileId,
-                                                @RequestParam(value = "unprocessed", required = false, defaultValue = FALSE) boolean unprocessed) {
+                                                @RequestParam(value = "unprocessed", required = false, defaultValue = FALSE) boolean unprocessed,
+                                                @RequestParam(value = "includeDictChanges", required = false, defaultValue = TRUE) boolean includeDictChanges) {
 
         accessControlService.checkDossierExistenceAndViewPermissionsToDossier(dossierId);
         accessControlService.validateFileResourceExistence(fileId);
-        return manualRedactionService.getManualRedactions(fileId, unprocessed);
+        return manualRedactionService.getManualRedactions(fileId,
+                                                          ManualChangesQueryOptions.builder().includeOnlyUnprocessed(unprocessed).includeDictChanges(includeDictChanges).build());
     }
 
 
@@ -149,12 +150,12 @@ public class ManualRedactionController implements ManualRedactionResource {
         var response = commentService.addComment(fileId, annotationId, CommentRequest.builder().user(KeycloakSecurity.getUserId()).text(addCommentRequest.getText()).build());
 
         auditPersistenceService.audit(AuditRequest.builder()
-                .userId(KeycloakSecurity.getUserId())
+                                              .userId(KeycloakSecurity.getUserId())
-                .objectId(fileId)
+                                              .objectId(fileId)
-                .category(AuditCategory.DOCUMENT.name())
+                                              .category(AuditCategory.DOCUMENT.name())
-                .message("Comment was added.")
+                                              .message("Comment was added.")
-                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
+                                              .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
-                .build());
+                                              .build());
 
         return new CommentResponse(String.valueOf(response.getId()));
     }
@@ -169,22 +170,21 @@ public class ManualRedactionController implements ManualRedactionResource {
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
         accessControlService.checkAccessPermissionsToDossier(dossierId);
         accessControlService.verifyFileIsNotApproved(dossierId, fileId);
-        if (addRedactionRequests.stream().anyMatch(AddRedactionRequestModel::isAddToAllDossiers)) {
+        if (addRedactionRequests.stream()
+                .anyMatch(AddRedactionRequestModel::isAddToAllDossiers)) {
             accessControlService.verifyUserIsApprover(dossierId);
         } else {
             accessControlService.verifyUserIsMemberOrApprover(dossierId);
         }
 
-        List<AddRedactionRequest> requests = manualRedactionMapper.toAddRedactionRequestList(dossierId, addRedactionRequests, dossier);
+        List<ManualAddResponse> responseList = manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests, dossier);
-
-        List<ManualAddResponse> responseList = manualRedactionService.addAddRedaction(dossierId, fileId, requests);
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
-                .userId(KeycloakSecurity.getUserId())
+                                                                               .userId(KeycloakSecurity.getUserId())
-                .objectId(fileId)
+                                                                               .objectId(fileId)
-                .category(AuditCategory.DOCUMENT.name())
+                                                                               .category(AuditCategory.DOCUMENT.name())
-                .message("Manual redaction was added.")
+                                                                               .message("Manual redaction was added.")
-                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
+                                                                               .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
-                .build()));
+                                                                               .build()));
 
         return responseList;
     }
@@ -199,22 +199,26 @@ public class ManualRedactionController implements ManualRedactionResource {
         var dossier = dossierManagementService.getDossierById(dossierId, false, false);
         accessControlService.checkAccessPermissionsToDossier(dossierId);
         accessControlService.verifyFileIsNotApproved(dossierId, fileId);
-        if (removeRedactionRequests.stream().anyMatch(RemoveRedactionRequestModel::isRemoveFromAllDossiers)) {
+        if (removeRedactionRequests.stream()
+                .anyMatch(RemoveRedactionRequestModel::isRemoveFromAllDossiers)) {
             accessControlService.verifyUserIsApprover(dossierId);
         } else {
             accessControlService.verifyUserIsMemberOrApprover(dossierId);
         }
 
-        List<RemoveRedactionRequest> requests = manualRedactionMapper.toRemoveRedactionRequestList(dossierId, fileId, dossier.getDossierTemplateId(), removeRedactionRequests, includeUnprocessed);
+        List<ManualAddResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId,
-        List<ManualAddResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId, fileId, requests);
+                                                                                         fileId,
+                                                                                         removeRedactionRequests,
+                                                                                         dossier.getDossierTemplateId(),
+                                                                                         includeUnprocessed);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
-                .userId(KeycloakSecurity.getUserId())
+                                                                               .userId(KeycloakSecurity.getUserId())
-                .objectId(fileId)
+                                                                               .objectId(fileId)
-                .category(AuditCategory.DOCUMENT.name())
+                                                                               .category(AuditCategory.DOCUMENT.name())
-                .message("Redaction was manually removed")
+                                                                               .message("Redaction was manually removed")
-                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
+                                                                               .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
-                .build()));
+                                                                               .build()));
 
         return responseList;
     }
@@ -229,21 +233,21 @@ public class ManualRedactionController implements ManualRedactionResource {
         accessControlService.verifyFileIsNotApproved(dossierId, fileId);
         accessControlService.verifyUserIsMemberOrApprover(dossierId);
 
-        List<ForceRedactionRequest> requests = manualRedactionMapper.toForceRedactionRequestList(forceRedactionRequests);
+        List<ManualAddResponse> responseList = manualRedactionService.addForceRedaction(dossierId, fileId, forceRedactionRequests);
-        List<ManualAddResponse> responseList = manualRedactionService.addForceRedaction(dossierId, fileId, requests);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
-                .userId(KeycloakSecurity.getUserId())
+                                                                               .userId(KeycloakSecurity.getUserId())
-                .objectId(fileId)
+                                                                               .objectId(fileId)
-                .category(AuditCategory.DOCUMENT.name())
+                                                                               .category(AuditCategory.DOCUMENT.name())
-                .message("Skipped redaction was forced to be redacted")
+                                                                               .message("Skipped redaction was forced to be redacted")
-                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
+                                                                               .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
-                .build()));
+                                                                               .build()));
 
         return responseList;
     }
 
 
+    @Deprecated(forRemoval = true)
     @PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
     public List<ManualAddResponse> legalBasisChangeBulk(@PathVariable(DOSSIER_ID) String dossierId,
                                                         @PathVariable(FILE_ID) String fileId,
@@ -253,17 +257,15 @@ public class ManualRedactionController implements ManualRedactionResource {
         accessControlService.verifyFileIsNotApproved(dossierId, fileId);
         accessControlService.verifyUserIsMemberOrApprover(dossierId);
 
-        List<LegalBasisChangeRequest> requests = manualRedactionMapper.toLegalBasisChangeRequestList(legalBasisChangeRequests);
+        List<ManualAddResponse> responseList = manualRedactionService.addLegalBasisChange(dossierId, fileId, legalBasisChangeRequests);
-
-        List<ManualAddResponse> responseList = manualRedactionService.addLegalBasisChange(dossierId, fileId, requests);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
-                .userId(KeycloakSecurity.getUserId())
+                                                                               .userId(KeycloakSecurity.getUserId())
-                .objectId(fileId)
+                                                                               .objectId(fileId)
-                .category(AuditCategory.DOCUMENT.name())
+                                                                               .category(AuditCategory.DOCUMENT.name())
-                .message("Legal basis reason was changed")
+                                                                               .message("Legal basis reason was changed")
-                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
+                                                                               .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
-                .build()));
+                                                                               .build()));
 
         return responseList;
     }
@@ -280,17 +282,19 @@ public class ManualRedactionController implements ManualRedactionResource {
         accessControlService.verifyFileIsNotApproved(dossierId, fileId);
         accessControlService.verifyUserIsMemberOrApprover(dossierId);
 
-        List<RecategorizationRequest> requests = manualRedactionMapper.toRecategorizationRequestList(dossierId, fileId, dossier.getDossierTemplateId(), recategorizationRequests, includeUnprocessed);
+        List<ManualAddResponse> responseList = manualRedactionService.addRecategorization(dossierId,
-
+                                                                                          fileId,
-        List<ManualAddResponse> responseList = manualRedactionService.addRecategorization(dossierId, fileId, requests);
+                                                                                          recategorizationRequests,
+                                                                                          dossier.getDossierTemplateId(),
+                                                                                          includeUnprocessed);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
-                .userId(KeycloakSecurity.getUserId())
+                                                                               .userId(KeycloakSecurity.getUserId())
-                .objectId(fileId)
+                                                                               .objectId(fileId)
-                .category(AuditCategory.DOCUMENT.name())
+                                                                               .category(AuditCategory.DOCUMENT.name())
-                .message("Entity was recategorized.")
+                                                                               .message("Entity was recategorized.")
-                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
+                                                                               .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
-                .build()));
+                                                                               .build()));
 
         return responseList;
     }
@@ -306,16 +310,15 @@ public class ManualRedactionController implements ManualRedactionResource {
         accessControlService.verifyFileIsNotApproved(dossierId, fileId);
         accessControlService.verifyUserIsMemberOrApprover(dossierId);
 
-        List<ResizeRedactionRequest> requests = manualRedactionMapper.toResizeRedactionRequestList(resizeRedactionRequests);
+        List<ManualAddResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, includeUnprocessed);
-        List<ManualAddResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, requests, includeUnprocessed);
 
         responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
-                .userId(KeycloakSecurity.getUserId())
+                                                                               .userId(KeycloakSecurity.getUserId())
-                .objectId(fileId)
+                                                                               .objectId(fileId)
-                .category(AuditCategory.DOCUMENT.name())
+                                                                               .category(AuditCategory.DOCUMENT.name())
-                .message("Skipped redaction was resized to be redacted")
+                                                                               .message("Skipped redaction was resized to be redacted")
-                .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
+                                                                               .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
-                .build()));
+                                                                               .build()));
 
         return responseList;
     }

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java

@@ -1,5 +1,6 @@
 package com.iqser.red.persistence.service.v1.external.api.impl.controller;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
@@ -18,6 +19,7 @@ import org.springframework.web.bind.annotation.RestController;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*;
 
@@ -47,10 +49,8 @@ public class MigrationStatusController implements MigrationStatusResource {
 
         var filesInErrorState = saasMigrationStatusPersistenceService.findAllByStatus(ERROR);
 
-        Map<String, String> errorCauses = new HashMap<>();
+        var errorCauses = filesInErrorState.stream()
-        filesInErrorState.forEach(errorFile -> {
+                .collect(Collectors.toMap(errorFile -> errorFile.getDossierId() + "/" + errorFile.getFileId(), SaasMigrationStatusEntity::getErrorCause));
-            errorCauses.put(errorFile.getFileId(), errorFile.getErrorCause());
-        });
 
         return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build();
     }

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java

@@ -51,6 +51,8 @@ public interface DictionaryResource {
     String MERGED = "/merged";
     String DELETE = "/delete";
 
+    String UPDATE_FLAG = "/updateFlag";
+
     String COLOR_REST_PATH = ExternalApi.BASE_PATH + "/color";
 
     String DOSSIER_ID_PARAMETER_NAME = "dossierId";
@@ -194,4 +196,14 @@ public interface DictionaryResource {
     @GetMapping(value = COLOR_REST_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     Colors getColors(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId);
 
-}
+
+    @ResponseStatus(HttpStatus.NO_CONTENT)
+    @PostMapping(value = DICTIONARY_REST_PATH + UPDATE_FLAG + TYPE_PATH_VARIABLE + DOSSIER_TEMPLATE_PATH_VARIABLE)
+    @Operation(summary = "Updates flags regarding to selected type, dossier and dossier template.", description = "None")
+    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Successfully updated the flags of the type."), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The entry type is not found.")})
+    void changeFlags(@PathVariable(TYPE_PARAMETER_NAME) String type,
+                     @PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
+                     @RequestParam(value = DOSSIER_ID_PARAMETER_NAME) String dossierId,
+                     @RequestParam(value = "addToDictionary") boolean addToDictionary);
+
+}

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ManualRedactionResource.java

@@ -46,6 +46,7 @@ public interface ManualRedactionResource {
     String COMMENT_ID_PATH_VARIABLE = "/{" + COMMENT_ID + "}";
 
     String FALSE = "false";
+    String TRUE = "true";
 
 
     @ResponseStatus(value = HttpStatus.NO_CONTENT)
@@ -59,7 +60,11 @@ public interface ManualRedactionResource {
 
 
     @ResponseStatus(value = HttpStatus.OK)
-    @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/comment/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + ANNOTATION_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/comment/add"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE
+                         + ANNOTATION_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Adds a comment to a redaction/redaction request", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     CommentResponse addComment(@PathVariable(DOSSIER_ID) String dossierId,
@@ -79,7 +84,10 @@ public interface ManualRedactionResource {
 
 
     @ResponseStatus(value = HttpStatus.OK)
-    @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk/redaction/add"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Adds a manual redaction", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     List<ManualAddResponse> addRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
@@ -88,7 +96,10 @@ public interface ManualRedactionResource {
 
 
     @ResponseStatus(value = HttpStatus.OK)
-    @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk/redaction/remove"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Removes the redactions list", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     List<ManualAddResponse> removeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
@@ -98,7 +109,10 @@ public interface ManualRedactionResource {
 
 
     @ResponseStatus(value = HttpStatus.OK)
-    @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/force" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk/redaction/force"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Forces the redactions list", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     List<ManualAddResponse> forceRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
@@ -106,8 +120,12 @@ public interface ManualRedactionResource {
                                                @RequestBody Set<ForceRedactionRequestModel> forceRedactionRequests);
 
 
+    @Deprecated(forRemoval = true)
     @ResponseStatus(value = HttpStatus.OK)
-    @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/legalBasisChange" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk/redaction/legalBasisChange"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Changes the legal basis reasons list", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     List<ManualAddResponse> legalBasisChangeBulk(@PathVariable(DOSSIER_ID) String dossierId,
@@ -116,7 +134,10 @@ public interface ManualRedactionResource {
 
 
     @ResponseStatus(value = HttpStatus.OK)
-    @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/recategorize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk/redaction/recategorize"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Recategorizes the list of redaction log entries", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     List<ManualAddResponse> recategorizeBulk(@PathVariable(DOSSIER_ID) String dossierId,
@@ -126,7 +147,10 @@ public interface ManualRedactionResource {
 
 
     @ResponseStatus(value = HttpStatus.OK)
-    @PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/resize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = MANUAL_REDACTION_REST_PATH
+                         + "/bulk/redaction/resize"
+                         + DOSSIER_ID_PATH_PARAM
+                         + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Resizes the redactions list", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier  or file not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     List<ManualAddResponse> resizeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
@@ -137,15 +161,24 @@ public interface ManualRedactionResource {
 
     @ResponseStatus(value = HttpStatus.OK)
     @GetMapping(value = MANUAL_REDACTION_REST_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
-    @Operation(summary = "Returns the manual redactions", description = "If the unprocessed flag is true then only the unprocessed manual redactions are returned. If the flag is false" + "all manual redactions are returned. Default value for the flag is false.")
+    @Operation(summary = "Returns the manual redactions", description = """
+            If the unprocessed flag is true then only the unprocessed manual redactions are returned. If the flag is false\
+            all manual redactions are returned. Default value for the flag is false.\
+            If the includeDictChanges flag is false, only local manual redactions will be returned, otherwise all are returned. Default value for the flag is true.
+            """)
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found")})
     ManualRedactions getManualRedactions(@PathVariable(DOSSIER_ID) String dossierId,
                                          @PathVariable(FILE_ID) String fileId,
-                                         @RequestParam(value = "unprocessed", required = false, defaultValue = FALSE) boolean unprocessed);
+                                         @RequestParam(value = "unprocessed", required = false, defaultValue = FALSE) boolean unprocessed,
+                                         @RequestParam(value = "includeDictChanges", required = false, defaultValue = TRUE) boolean includeDictChanges);
 
 
     @ResponseStatus(value = HttpStatus.OK)
-    @GetMapping(value = MANUAL_REDACTION_REST_PATH + "/comments" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + ANNOTATION_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
+    @GetMapping(value = MANUAL_REDACTION_REST_PATH
+                        + "/comments"
+                        + DOSSIER_ID_PATH_PARAM
+                        + FILE_ID_PATH_VARIABLE
+                        + ANNOTATION_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(summary = "Returns the comments for a specific annotation in a specific file", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Dossier or file not found")})
     AnnotationComments getComments(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @PathVariable(ANNOTATION_ID) String annotationId);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/custom/service/CustomPermissionService.java

@@ -40,10 +40,10 @@ public class CustomPermissionService {
 
     public void syncAllCustomPermissions() {
 
-        log.info("Syncing all custom permissions");
+        log.debug("Syncing all custom permissions");
         var targetObjects = getAllSupportedTargetObjects();
         targetObjects.forEach(this::applyCustomPermissions);
-        log.info("All custom permissions synced");
+        log.debug("All custom permissions synced");
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java

@@ -14,6 +14,7 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+@Deprecated(forRemoval = true)
 @Data
 @Builder
 @AllArgsConstructor

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRecategorizationEntity.java

@@ -47,6 +47,12 @@ public class ManualRecategorizationEntity implements IBaseAnnotation {
     private boolean addToDictionary;
     @Column
     private boolean addToAllDossiers;
+    @Column(length = 4000)
+    private String legalBasis;
+    @Column(length = 1024)
+    private String section;
+    @Column
+    private String value;
 
     @ManyToOne
     private FileEntity fileStatus;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java

@@ -44,7 +44,7 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation {
     private String value;
     @Column(length = 4000)
     private String reason;
-    @Column
+    @Column(length = 4000)
     private String legalBasis;
     @Column
     private String section;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/RankDeDuplicationService.java

@@ -0,0 +1,93 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class RankDeDuplicationService {
+
+    private final DossierTemplateManagementService dossierTemplateManagementService;
+
+    private final DictionaryPersistenceService dictionaryPersistenceService;
+
+
+    @SneakyThrows
+    public void deduplicate() {
+
+        dossierTemplateManagementService.getAllDossierTemplates()
+                .stream()
+                .map(DossierTemplate::getId)
+                .forEach(this::deduplicate);
+    }
+
+
+    private void deduplicate(String dossierTemplateId) {
+
+        log.info("deduplicate for dossierTemplateID: {}", dossierTemplateId);
+        List<TypeEntity> allDossierTemplateTypes = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false);
+        TreeMap<Integer, List<TypeEntity>> typesSortedByRank = new TreeMap<>(allDossierTemplateTypes.stream()
+                                                                                     .collect(Collectors.groupingBy(TypeEntity::getRank, Collectors.toList())));
+
+        var lastRankOk = -1;
+        // we create a map with every rank that needs to be updated and the value is the starting point for that rank
+        Map<Integer, Integer> rankToNewRankUpdateMap = new TreeMap<>(Collections.reverseOrder());
+        for (var typesByRankEntry : typesSortedByRank.entrySet()) {
+            log.debug("Rank: {} - number of types {}", typesByRankEntry.getKey(), typesByRankEntry.getValue().size());
+            typesByRankEntry.getValue()
+                    .forEach(t -> log.debug("type: {} - dtId: {} - dId: {} typeId: {}", t.getType(), t.getDossierTemplateId(), t.getDossierId(), t.getId()));
+
+            if (typesByRankEntry.getValue().size() > 1) {
+                if (typesByRankEntry.getKey() > lastRankOk) {
+                    rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), typesByRankEntry.getKey());
+                    lastRankOk = typesByRankEntry.getKey() + typesByRankEntry.getValue().size() - 1;
+                } else {
+                    rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), lastRankOk + 1);
+                    lastRankOk = lastRankOk + typesByRankEntry.getValue().size();
+                }
+            } else {
+                if (typesByRankEntry.getKey() > lastRankOk) {
+                    lastRankOk = typesByRankEntry.getKey();
+                } else {
+                    lastRankOk = lastRankOk + 1;
+                    rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), lastRankOk);
+                }
+            }
+        }
+        // need to update in reverse order so we don't get exception for duplicate ranks
+        for (var rankToUpdateEntry : rankToNewRankUpdateMap.entrySet()) {
+            var newRank = rankToUpdateEntry.getValue();
+            for (TypeEntity t : typesSortedByRank.get(rankToUpdateEntry.getKey())) {
+
+                if (newRank != t.getRank()) {
+                    log.info("Type {} with rank: {} will be updated to rank: {} - typeId {}", t.getType(), t.getRank(), newRank, t.getId());
+                    dictionaryPersistenceService.updateRankForType(t.getId(), newRank);
+                    var dossierTypes = dictionaryPersistenceService.getAllDossierTypesForDossierTemplateAndType(dossierTemplateId, t.getType(), false);
+
+                    for (TypeEntity td : dossierTypes) {
+                        log.info("Type {} with rank: {} will be updated to rank: {} - typeId {}", td.getType(), td.getRank(), newRank, td.getId());
+                        dictionaryPersistenceService.updateRankForType(td.getId(), newRank);
+                    }
+
+                }
+                newRank = newRank + 1;
+            }
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java

@@ -0,0 +1,151 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+
+import org.springframework.beans.BeanUtils;
+import org.springframework.stereotype.Service;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RecategorizationPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class SaasMigrationManualChangesUpdateService {
+
+    private final AddRedactionPersistenceService addRedactionPersistenceService;
+    private final RemoveRedactionPersistenceService removeRedactionPersistenceService;
+    private final ResizeRedactionPersistenceService resizeRedactionPersistenceService;
+
+    private final HashFunction hashFunction = Hashing.murmur3_128();
+
+
+    public void convertUnprocessedAddToDictionariesToLocalChanges(String fileId) {
+
+        var unprocessedManualAdds = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly());
+        for (var unprocessedManualAdd : unprocessedManualAdds) {
+
+            if (!unprocessedManualAdd.getDictionaryEntryType().equals(DictionaryEntryType.ENTRY)) {
+                continue;
+            }
+
+            if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) {
+                // copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table.
+                // It has no functionality, its only there, such that the UI can show a pending change.
+                ManualRedactionEntryEntity pendingDictAdd = ManualRedactionEntryEntity.builder()
+                        .id(buildSecondaryId(unprocessedManualAdd.getId(), fileId))
+                        .user(unprocessedManualAdd.getUser())
+                        .typeId(unprocessedManualAdd.getTypeId())
+                        .value(unprocessedManualAdd.getValue())
+                        .reason(unprocessedManualAdd.getReason())
+                        .legalBasis(unprocessedManualAdd.getLegalBasis())
+                        .section(unprocessedManualAdd.getSection())
+                        .rectangle(unprocessedManualAdd.isRectangle())
+                        .addToDictionary(unprocessedManualAdd.isAddToDictionary())
+                        .addToAllDossiers(unprocessedManualAdd.isAddToAllDossiers())
+                        .dictionaryEntryType(DictionaryEntryType.ENTRY)
+                        .requestDate(unprocessedManualAdd.getRequestDate())
+                        .positions(new ArrayList<>(unprocessedManualAdd.getPositions())) // copy to new List
+                        .fileStatus(unprocessedManualAdd.getFileStatus())
+                        .textBefore(unprocessedManualAdd.getTextBefore())
+                        .textAfter(unprocessedManualAdd.getTextAfter())
+                        .sourceId(unprocessedManualAdd.getSourceId())
+                        .typeIdsOfModifiedDictionaries(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries())
+                        .build();
+
+                addRedactionPersistenceService.updateOrCreate(pendingDictAdd);
+
+                // change existing dict add to unprocessed manual add. ID must match with prior entry, such that other unprocessed manual changes may be applied to it.
+                unprocessedManualAdd.setAddToDictionary(false);
+                unprocessedManualAdd.setAddToAllDossiers(false);
+                unprocessedManualAdd.setLegalBasis("");
+                unprocessedManualAdd.setTypeIdsOfModifiedDictionaries(Collections.emptySet());
+                unprocessedManualAdd.setDictionaryEntryType(null);
+
+                addRedactionPersistenceService.updateOrCreate(unprocessedManualAdd);
+            }
+        }
+    }
+
+
+    public void convertUnprocessedRemoveFromDictionariesToLocalChanges(String fileId) {
+
+        var unprocessedManualRemoves = removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly());
+        for (var unprocessedManualRemove : unprocessedManualRemoves) {
+            if (unprocessedManualRemove.isRemoveFromDictionary() || unprocessedManualRemove.isRemoveFromAllDossiers()) {
+
+                IdRemovalEntity pendingDictRemoval = IdRemovalEntity.builder()
+                        .id(buildSecondaryId(unprocessedManualRemove.getId(), fileId))
+                        .user(unprocessedManualRemove.getUser())
+                        .removeFromDictionary(unprocessedManualRemove.isRemoveFromDictionary())
+                        .removeFromAllDossiers(unprocessedManualRemove.isRemoveFromAllDossiers())
+                        .requestDate(unprocessedManualRemove.getRequestDate())
+                        .page(unprocessedManualRemove.getPage())
+                        .fileStatus(unprocessedManualRemove.getFileStatus())
+                        .typeIdsOfModifiedDictionaries(unprocessedManualRemove.getTypeIdsOfModifiedDictionaries())
+                        .build();
+
+                removeRedactionPersistenceService.updateOrCreate(pendingDictRemoval);
+
+                unprocessedManualRemove.setRemoveFromDictionary(false);
+                unprocessedManualRemove.setRemoveFromAllDossiers(false);
+                unprocessedManualRemove.setTypeIdsOfModifiedDictionaries(Collections.emptySet());
+
+                removeRedactionPersistenceService.updateOrCreate(unprocessedManualRemove);
+            }
+        }
+    }
+
+
+    public void convertUnprocessedResizeWithDictionariesToLocalChanges(String fileId) {
+
+        var unprocessedManualResizes = resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, ManualChangesQueryOptions.unprocessedOnly());
+        for (var unprocessedManualResize : unprocessedManualResizes) {
+            if (unprocessedManualResize.getUpdateDictionary() || unprocessedManualResize.isAddToAllDossiers()) {
+
+                ManualResizeRedactionEntity pendingDictResize = ManualResizeRedactionEntity.builder()
+                        .id(buildSecondaryId(unprocessedManualResize.getId(), fileId))
+                        .user(unprocessedManualResize.getUser())
+                        .requestDate(unprocessedManualResize.getRequestDate())
+                        .page(unprocessedManualResize.getPage())
+                        .value(unprocessedManualResize.getValue())
+                        .updateDictionary(unprocessedManualResize.getUpdateDictionary())
+                        .fileStatus(unprocessedManualResize.getFileStatus())
+                        .positions(new ArrayList<>(unprocessedManualResize.getPositions()))
+                        .textBefore(unprocessedManualResize.getTextBefore())
+                        .textAfter(unprocessedManualResize.getTextAfter())
+                        .addToAllDossiers(unprocessedManualResize.isAddToAllDossiers())
+                        .typeIdsOfModifiedDictionaries(unprocessedManualResize.getTypeIdsOfModifiedDictionaries())
+                        .build();
+
+                resizeRedactionPersistenceService.updateOrCreate(pendingDictResize);
+
+                unprocessedManualResize.setUpdateDictionary(false);
+                unprocessedManualResize.setAddToAllDossiers(false);
+                unprocessedManualResize.setTypeIdsOfModifiedDictionaries(Collections.emptySet());
+
+                resizeRedactionPersistenceService.updateOrCreate(unprocessedManualResize);
+            }
+        }
+    }
+
+
+    private AnnotationEntityId buildSecondaryId(AnnotationEntityId annotationEntityId, String fileId) {
+
+        return new AnnotationEntityId(hashFunction.hashString(annotationEntityId.getAnnotationId(), StandardCharsets.UTF_8).toString(), fileId);
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java

@@ -1,22 +1,36 @@
 package com.iqser.red.service.persistence.management.v1.processor.migration;
 
+import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE;
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
+import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
 import com.iqser.red.service.persistence.management.v1.processor.service.IndexingService;
 import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
 import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
+import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
 import com.iqser.red.service.redaction.v1.model.MigrationRequest;
 import com.iqser.red.storage.commons.exception.StorageException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
@@ -32,17 +46,6 @@ import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -62,6 +65,10 @@ public class SaasMigrationService implements TenantSyncService {
     StorageService storageService;
     SaasAnnotationIdMigrationService saasAnnotationIdMigrationService;
     UncompressedFilesMigrationService uncompressedFilesMigrationService;
+    ManualRedactionService manualRedactionService;
+    CommentService commentService;
+    RankDeDuplicationService rankDeDuplicationService;
+    SaasMigrationManualChangesUpdateService saasMigrationManualChangesUpdateService;
 
 
     @Override
@@ -82,47 +89,36 @@ public class SaasMigrationService implements TenantSyncService {
         uncompressedFilesMigrationService.migrateUncompressedFiles(tenantId);
         log.info("Finished uncompressed files migration ...");
 
+        rankDeDuplicationService.deduplicate();
         int numberOfFiles = 0;
-        var dossiers = dossierService.getAllDossiers()
+
-                .stream()
+        var files = saasMigrationStatusPersistenceService.findAll();
-                .filter(dossier -> dossier.getHardDeletedTime() == null)
+
-                .toList();
+        for (var file : files) {
-        for (var dossier : dossiers) {
+
-            var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId())
+            if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) {
-                    .stream()
+                log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus());
-                    .filter(file -> file.getHardDeletedTime() == null)
+                continue;
-                    .toList();
-            var migrationStati = saasMigrationStatusPersistenceService.findAll()
-                    .stream()
-                    .collect(Collectors.toMap(SaasMigrationStatusEntity::getFileId, SaasMigrationStatusEntity::getStatus));
-            for (var file : files) {
-                if (notExistsOrError(file, migrationStati)) {
-                    // delete NER_ENTITIES since offsets depend on old document structure.
-                    storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossier.getId(), file.getId(), FileType.NER_ENTITIES));
-                    saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossier.getId(), file.getId());
-                    var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getId(), file.getId(), false);
-                    rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
-                    numberOfFiles++;
-                } else {
-                    log.info("Skipping file with id {} and dossier {}, since it is already migrated or is currently being migrated!", file.getId(), file.getDossierId());
-                }
             }
+
+            // delete NER_ENTITIES since offsets depend on old document structure.
+            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(file.getDossierId(), file.getFileId(), FileType.NER_ENTITIES));
+
+            var layoutParsingRequest = layoutParsingRequestFactory.build(file.getDossierId(), file.getFileId(), false);
+
+            rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
+
+            numberOfFiles++;
+
         }
 
         log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId());
-
         if (numberOfFiles == 0) {
             finalizeMigration();
         }
     }
 
 
-    private boolean notExistsOrError(FileEntity file, Map<String, SaasMigrationStatus> migrationStati) {
-
-        return migrationStati.getOrDefault(file.getId(), SaasMigrationStatus.ERROR).equals(SaasMigrationStatus.ERROR);
-    }
-
-
     public void startMigrationForFile(String dossierId, String fileId) {
 
         log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId);
@@ -148,20 +144,27 @@ public class SaasMigrationService implements TenantSyncService {
             return;
         }
 
+        log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
         saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.DOCUMENT_FILES_MIGRATED);
 
+        if (fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
+            saasMigrationManualChangesUpdateService.convertUnprocessedAddToDictionariesToLocalChanges(fileId);
+        }
+
         try {
             indexingService.reindex(dossierId, Set.of(fileId), false);
 
             String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId();
+
             rabbitTemplate.convertAndSend(MIGRATION_QUEUE,
                                           MigrationRequest.builder()
                                                   .dossierTemplateId(dossierTemplateId)
                                                   .dossierId(dossierId)
                                                   .fileId(fileId)
-                                                  .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
+                                                  .fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED))
+                                                  .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
+                                                  .entitiesWithComments(commentService.getCommentCounts(fileId).keySet())
                                                   .build());
-            log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
         } catch (Exception e) {
             log.error("Queuing of entityLog migration failed with {}", e.getMessage());
             saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage()));
@@ -188,7 +191,7 @@ public class SaasMigrationService implements TenantSyncService {
         saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.REDACTION_LOGS_MIGRATED);
 
         log.info("EntityLog migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
-        migrateAnnotationIds(dossierId, fileId);
+        migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(dossierId, fileId);
     }
 
 
@@ -215,11 +218,15 @@ public class SaasMigrationService implements TenantSyncService {
     }
 
 
-    private void migrateAnnotationIds(String dossierId, String fileId) {
+    private void migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(String dossierId, String fileId) {
 
         MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
         Map<String, String> oldToNewMapping = migratedIds.buildOldToNewMapping();
         updateAnnotationIds(dossierId, fileId, oldToNewMapping);
+        List<ManualRedactionEntry> manualRedactionEntriesToAdd = migratedIds.getManualRedactionEntriesToAdd();
+        int count = addManualRedactionEntries(manualRedactionEntriesToAdd);
+        log.info("Added {} additional manual entries.", count);
+        deleteSectionGridAndNerEntitiesFiles(dossierId, fileId);
         saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
 
         log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
@@ -227,16 +234,51 @@ public class SaasMigrationService implements TenantSyncService {
     }
 
 
+    private void deleteSectionGridAndNerEntitiesFiles(String dossierId, String fileId) {
+
+        try {
+            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
+        } catch (StorageObjectDoesNotExist e) {
+            log.info("No sectiongrid found for {}, {}, ignoring....", dossierId, fileId);
+        }
+
+        try {
+            storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
+        } catch (StorageObjectDoesNotExist e) {
+            log.info("No ner entities file found for {}, {}, ignoring....", dossierId, fileId);
+        }
+    }
+
+
+    private int addManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
+
+        manualRedactionEntriesToAdd.forEach(add -> {
+            if (add.getSection() != null && add.getSection().length() > 254) {
+                add.setSection(add.getSection().substring(0, 254));
+            }
+        });
+
+        return manualRedactionService.addManualRedactionEntries(manualRedactionEntriesToAdd, true);
+    }
+
+
     public void revertMigrationForFile(String dossierId, String fileId) {
 
         log.info("Reverting Migration for dossierId {} and fileId {}", dossierId, fileId);
         MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
         Map<String, String> newToOldMapping = migratedIds.buildNewToOldMapping();
         updateAnnotationIds(dossierId, fileId, newToOldMapping);
+        deleteManualRedactionEntries(migratedIds.getManualRedactionEntriesToAdd());
         saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
     }
 
 
+    private void deleteManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd) {
+
+        manualRedactionService.deleteManualRedactionEntries(manualRedactionEntriesToAdd);
+    }
+
+
     private void updateAnnotationIds(String dossierId, String fileId, Map<String, String> idMapping) {
 
         try {
@@ -257,7 +299,7 @@ public class SaasMigrationService implements TenantSyncService {
     private void finalizeMigration() {
 
         if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
-            automaticAnalysisJob.startForTenant(TenantContext.getTenantId());
+//            automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
             tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build());
             log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/UncompressedFilesMigrationService.java

@@ -141,7 +141,7 @@ public class UncompressedFilesMigrationService {
             } while (!keysToMigrate.isEmpty() && attempts <= 3);
 
             if (!keysToMigrate.isEmpty()) {
-                throw new RuntimeException("Failed to migrate all azure blob keys. Remaining: " + keysToMigrate.size());
+                log.error("Failed to migrate {},", keysToMigrate.size());
             }
 
         }
@@ -191,6 +191,7 @@ public class UncompressedFilesMigrationService {
                 log.info("Key: {} migrated successfully", key);
 
             } catch (Exception e) {
+                log.error("Failed to migrate {}", key, e);
                 failedKeys.add(key);
             }
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeRenameMigration15.java

@@ -0,0 +1,101 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import java.util.Set;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Setter
+@Service
+public class ManualRedactionTypeRenameMigration15 extends Migration {
+
+    @Autowired
+    private FileStatusPersistenceService fileStatusPersistenceService;
+
+    @Autowired
+    private FileManagementStorageService fileManagementStorageService;
+
+    @Autowired
+    private ManualRedactionService manualRedactionService;
+
+
+    public ManualRedactionTypeRenameMigration15() {
+
+        super(NAME, VERSION);
+    }
+
+
+    private static final String NAME = "Migrate ManualRedactionType names in EntityLog";
+    private static final long VERSION = 15;
+
+
+    @Override
+    /*
+     * Migrates the ManualRedactionType ADD_LOCALLY -> ADD, FORCE_REDACT -> FORCE, FORCE_HINT -> FORCE, REMOVE_LOCALLY -> REMOVE.
+     */
+    protected void migrate() {
+
+        log.info("Migrating ManualRedactionType names");
+
+        Set<Class<? extends BaseAnnotation>> excludedClasses = Set.of(ManualResizeRedaction.class, ManualRecategorization.class, ManualLegalBasisChange.class);
+        ManualChangesQueryOptions options = ManualChangesQueryOptions.builder().includeDictChanges(false).excludedClasses(excludedClasses).build();
+
+        var files = fileStatusPersistenceService.getAllFiles();
+        for (FileEntity file : files) {
+
+            ManualRedactions manualChanges = manualRedactionService.getManualRedactions(file.getId(), options);
+
+            if (manualChanges.isEmpty()) {
+                continue;
+            }
+
+            if (!fileManagementStorageService.objectExists(file.getDossierId(), file.getId(), FileType.ENTITY_LOG)) {
+                continue;
+            }
+
+            log.info("Migrating ManualRedactionType names for fileId {}", file.getId());
+
+            EntityLog entityLog = fileManagementStorageService.getEntityLog(file.getDossierId(), file.getId());
+            for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) {
+                for (ManualChange manualChange : entityLogEntry.getManualChanges()) {
+                    if (manualChange.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY)) {
+                        manualChange.setManualRedactionType(ManualRedactionType.ADD);
+                    }
+                    if (manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT)) {
+                        manualChange.setManualRedactionType(ManualRedactionType.FORCE);
+                    }
+                    if (manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT)) {
+                        manualChange.setManualRedactionType(ManualRedactionType.FORCE);
+                    }
+                    if (manualChange.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY)) {
+                        manualChange.setManualRedactionType(ManualRedactionType.REMOVE);
+                    }
+                }
+            }
+            fileManagementStorageService.storeJSONObject(file.getDossierId(), file.getId(), FileType.ENTITY_LOG, entityLog);
+        }
+
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java

@@ -0,0 +1,89 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Setter
+@Service
+public class MigrateImportedRedactionsFiles17 extends Migration {
+
+    @Autowired
+    private FileStatusPersistenceService fileStatusPersistenceService;
+
+    @Autowired
+    private FileManagementStorageService fileManagementStorageService;
+
+    private static final String NAME = "Migrating the ImportedRedaction files to the new structure";
+    private static final long VERSION = 17;
+
+
+    public MigrateImportedRedactionsFiles17() {
+
+        super(NAME, VERSION);
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        var files = fileStatusPersistenceService.getAllFiles();
+        for (FileEntity file : files) {
+
+            if (fileManagementStorageService.objectExists(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS)) {
+
+                com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions oldImportedRedactions = fileManagementStorageService.getImportedRedactions(
+                        file.getDossierId(),
+                        file.getId());
+
+                fileManagementStorageService.deleteObject(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS);
+
+                Map<Integer, List<ImportedRedaction>> importedRedactionsPerPage = oldImportedRedactions.getImportedRedactions().entrySet()
+                        .stream()
+                        .collect(Collectors.toMap(Map.Entry::getKey,
+                                                  entry -> entry.getValue()
+                                                          .stream()
+                                                          .map(this::toImportedRedaction)
+                                                          .toList()));
+
+                ImportedRedactionsPerPage importedRedactions = new ImportedRedactionsPerPage(importedRedactionsPerPage);
+
+                fileManagementStorageService.storeJSONObject(file.getDossierId(), file.getId(), FileType.IMPORTED_REDACTIONS, importedRedactions);
+            }
+        }
+    }
+
+
+    private ImportedRedaction toImportedRedaction(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction importedRedaction) {
+
+        return ImportedRedaction.builder().id(importedRedaction.getId()).positions(mapPositions(importedRedaction)).build();
+    }
+
+
+    private static List<Position> mapPositions(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction importedRedaction) {
+
+        return importedRedaction.getPositions()
+                .stream()
+                .map(rectangle -> new Position(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage()))
+                .toList();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RankDeDuplicationMigration16.java

@@ -0,0 +1,41 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService;
+
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Setter
+@Service
+public class RankDeDuplicationMigration16 extends Migration {
+
+    @Autowired
+    private RankDeDuplicationService rankDeDuplicationService;
+
+
+    public RankDeDuplicationMigration16() {
+
+        super(NAME, VERSION);
+    }
+
+
+    private static final String NAME = "Adding to the migration the rank de-duplication";
+    private static final long VERSION = 16;
+
+
+    @Override
+    /*
+     * In cases were we have duplicate ranks for entities, this needs to be fixed
+     */ protected void migrate() {
+
+        log.info("Migration: Checking for duplicate ranks");
+        rankDeDuplicationService.deduplicate();
+
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/ManualChangesQueryOptions.java

@@ -0,0 +1,51 @@
+package com.iqser.red.service.persistence.management.v1.processor.model;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+
+import lombok.Builder;
+import lombok.Getter;
+
+@Getter
+@Builder
+@SuppressWarnings("checkstyle:all") // explicit initialization of false. IMO it improves readability here.
+public class ManualChangesQueryOptions {
+
+    @Builder.Default
+    private boolean includeOnlyUnprocessed = false;
+    @Builder.Default
+    private boolean includeDictChanges = true;
+    @Builder.Default
+    private boolean includeDeletions = false;
+    @Builder.Default
+    private Set<Class<? extends BaseAnnotation>> excludedClasses = Collections.emptySet();
+
+
+    public static ManualChangesQueryOptions allWithoutDeleted() {
+
+        return ManualChangesQueryOptions.builder().includeDeletions(false).includeDictChanges(true).includeOnlyUnprocessed(false).build();
+    }
+
+
+    public static ManualChangesQueryOptions all() {
+
+        return ManualChangesQueryOptions.builder().includeDeletions(true).includeDictChanges(true).includeOnlyUnprocessed(false).build();
+    }
+
+
+    public static ManualChangesQueryOptions unprocessedOnly() {
+
+        return ManualChangesQueryOptions.builder().includeDeletions(false).includeDictChanges(true).includeOnlyUnprocessed(true).build();
+    }
+
+
+    public static ManualChangesQueryOptions localChangesOnly() {
+
+        return ManualChangesQueryOptions.builder().includeDeletions(false).includeDictChanges(false).includeOnlyUnprocessed(false).build();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java

@@ -157,11 +157,10 @@ public class AnalysisFlagsCalculationService {
         return !entryType.equals(EntryType.HINT) && //
                !entryType.equals(EntryType.RECOMMENDATION) && //
                StringUtils.isNotEmpty(entry.getReason()) && //
-               (manualChange.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY) || //
+               (manualChange.getManualRedactionType().equals(ManualRedactionType.ADD) || //
                 manualChange.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE) || //
-                manualChange.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY) || //
+                manualChange.getManualRedactionType().equals(ManualRedactionType.REMOVE) || //
-                manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || //
+                manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE) || //
-                manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT) || //
                 manualChange.getManualRedactionType().equals(ManualRedactionType.LEGAL_BASIS_CHANGE) || //
                 manualChange.getManualRedactionType().equals(ManualRedactionType.RESIZE)) && //
                manualChange.getProcessedDate() != null && //

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java

@@ -429,4 +429,15 @@ public class DictionaryService {
         return MagicConverter.convert(colorsService.getColors(dossierTemplateId), Colors.class);
     }
 
+
+    public void changeAddToDictionary(String type, String dossierTemplateId, String dossierId, boolean addToDictionary) {
+
+        var typeEntity = dictionaryPersistenceService.getType(toTypeId(type, dossierTemplateId, dossierId));
+        if (typeEntity.isDossierDictionaryOnly()) {
+            typeEntity.setAddToDictionaryAction(addToDictionary);
+            dictionaryPersistenceService.saveType(typeEntity);
+        }
+
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java

@@ -39,7 +39,6 @@ import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
-import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -77,6 +76,7 @@ public class DossierTemplateCloneService {
 
         dossierTemplateRepository.findById(dossierTemplateId).ifPresentOrElse(dossierTemplate -> {
 
+            dossierTemplatePersistenceService.validateDossierTemplateForDuplicateRanks(dossierTemplateId);
             OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
             clonedDossierTemplate.setId(UUID.randomUUID().toString());
             clonedDossierTemplate.setName(cloneDossierTemplateRequest.getName());

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java

@@ -5,12 +5,16 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ty
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -23,12 +27,14 @@ import com.iqser.red.service.persistence.management.v1.processor.configuration.M
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.PendingDictionaryEntryFactory;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
@@ -49,57 +55,181 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 
 import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Service
 @Slf4j
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityLogMergeService {
 
-    private final DictionaryPersistenceService dictionaryPersistenceService;
+    DictionaryPersistenceService dictionaryPersistenceService;
-    private final RabbitTemplate rabbitTemplate;
+    RabbitTemplate rabbitTemplate;
-    private final FileStatusService fileStatusService;
+    FileStatusService fileStatusService;
-    private final FileStatusPersistenceService fileStatusPersistenceService;
+    FileStatusPersistenceService fileStatusPersistenceService;
+    PendingDictionaryEntryFactory pendingDictionaryEntryFactory;
 
 
     @Observed(name = "EntityLogMergeService", contextualName = "merge-entity-log")
-    public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier) {
+    public EntityLog mergeEntityLog(ManualRedactions unprocessedManualRedactions, EntityLog entityLog, DossierEntity dossier) {
 
         log.debug("Merging EntityLog");
-        List<BaseAnnotation> allManualChanges = allManualChanges(manualRedactions);
+
-        List<String> manualChangesIds = allManualChanges.stream().map(BaseAnnotation::getAnnotationId).toList();
+        long start = System.currentTimeMillis();
-        List<EntityLogEntry> matchingEntities = entityLog.getEntityLogEntry()
-                .stream()
-                .filter(entityLogEntry -> manualChangesIds.contains(entityLogEntry.getId()))
-                .collect(Collectors.toList());
         final int analysisNumber = entityLog.getAnalysisNumber();
 
-        // Sort manual changes by date, so we process them in order of when they were requested
+        Map<String, List<BaseAnnotation>> allManualChanges = unprocessedManualRedactions.buildAll()
-        allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList();
+                .stream()
-        allManualChanges.forEach(manualChange -> {
+                .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
-            // this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher
-            if (manualChange instanceof ManualRedactionEntry manualRedactionEntry) {
-                var entityLogEntry = mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier);
-                entityLogEntry.ifPresent(matchingEntities::add);
-            } else if (manualChange instanceof IdRemoval idRemoval) {
-                mergeIdsToRemove(idRemoval, matchingEntities, analysisNumber);
-            } else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
-                mergeResizeRedactions(manualResizeRedaction, matchingEntities, analysisNumber);
-            } else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
-                mergeLegalBasisChanges(manualLegalBasisChange, matchingEntities, analysisNumber);
-            } else if (manualChange instanceof ManualRecategorization manualRecategorization) {
-                mergeRecategorizations(manualRecategorization, matchingEntities, dossier, analysisNumber);
-            } else if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
-                mergeForceRedactions(manualForceRedaction, matchingEntities, analysisNumber);
-            }
-        });
 
-        log.debug("EntityLog merged successfully.");
+        List<EntityLogEntry> entityLogEntries = new LinkedList<>(entityLog.getEntityLogEntry());
+
+        Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions,
+                                                                                            entityLog,
+                                                                                            dossier).collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
+
+        entityLogEntries.addAll(addedLocalManualEntries.values());
+
+
+        buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(entityLogEntries::add);
+
+        int numberOfAddedPendingEntries = 0; // since list is dynamically growing we need to keep track of the number of added pending entries to ignore them in the loop
+        for (int i = 0; i + numberOfAddedPendingEntries < entityLogEntries.size(); i++) {
+
+            EntityLogEntry entityLogEntry = entityLogEntries.get(i + numberOfAddedPendingEntries);
+
+            if (isDuplicatedByAny(entityLogEntry, addedLocalManualEntries)) {
+
+                mergeOriginalAndLocallyAddedEntries(addedLocalManualEntries, entityLogEntry, analysisNumber);
+                continue;
+            }
+
+            if (allManualChanges.containsKey(entityLogEntry.getId())) {
+
+                List<EntityLogEntry> pendingImageRecategorizations = mergeLocalManualChangesAndReturnNonMergeableAsPending(dossier,
+                                                                                                                           allManualChanges,
+                                                                                                                           entityLogEntry,
+                                                                                                                           analysisNumber);
+
+                List<EntityLogEntry> pendingDictionaryEntries = buildPendingDictionaryEntries(allManualChanges, entityLogEntry);
+
+                // insert pending entries directly after the associated entry to enable performant linking in UI
+                for (EntityLogEntry pendingDictionaryEntry : concatLists(pendingDictionaryEntries, pendingImageRecategorizations)) {
+                    numberOfAddedPendingEntries++;
+                    entityLogEntries.add(i + numberOfAddedPendingEntries, pendingDictionaryEntry);
+                }
+
+            }
+
+        }
+
+        entityLog.setEntityLogEntry(entityLogEntries);
+
+        log.debug("EntityLog merged successfully in {} ms.", System.currentTimeMillis() - start);
         return entityLog;
     }
 
 
+    private static void mergeOriginalAndLocallyAddedEntries(Map<String, EntityLogEntry> addedLocalManualEntries, EntityLogEntry entityLogEntry, int analysisNumber) {
+
+        EntityLogEntry duplicateManualEntry = addedLocalManualEntries.get(entityLogEntry.getId());
+        duplicateManualEntry.getEngines().addAll(entityLogEntry.getEngines());
+
+        // Mark existing entry REMOVED, will be replaced by the manual one.
+        entityLogEntry.setState(EntryState.REMOVED);
+        entityLogEntry.setReason("Removed due to duplicate locally added manual entity.");
+        entityLogEntry.getChanges().add(Change.builder().type(ChangeType.REMOVED).dateTime(OffsetDateTime.now()).analysisNumber(analysisNumber).build());
+    }
+
+
+    private static List<EntityLogEntry> concatLists(List<EntityLogEntry> pendingDictionaryEntries, List<EntityLogEntry> pendingImageRecategorizations) {
+
+        return Stream.of(pendingDictionaryEntries, pendingImageRecategorizations)
+                .flatMap(Collection::stream)
+                .toList();
+    }
+
+
+    private Stream<EntityLogEntry> buildPendingDictionaryChanges(ManualRedactions unprocessedManualRedactions) {
+
+        return unprocessedManualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(manualAdd -> !manualAdd.isLocal())
+                .map(pendingDictionaryEntryFactory::buildAddToDictionaryEntry);
+    }
+
+
+    private Stream<EntityLogEntry> buildUnprocessedLocalManualRedactions(ManualRedactions unprocessedManualRedactions, EntityLog entityLog, DossierEntity dossier) {
+
+        return unprocessedManualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(ManualRedactionEntry::isLocal)
+                .map(manualRedactionEntry -> mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    private List<EntityLogEntry> buildPendingDictionaryEntries(Map<String, List<BaseAnnotation>> allManualChanges, EntityLogEntry entityLogEntry) {
+
+        return allManualChanges.getOrDefault(entityLogEntry.getId(), Collections.emptyList())
+                .stream()
+                .filter(baseAnnotation -> !baseAnnotation.isLocal())
+                .sorted(Comparator.comparing(BaseAnnotation::getRequestDate))
+                .map(dictionaryChange -> {
+                    if (dictionaryChange instanceof ManualRedactionEntry) {
+                        return null; // pending dictionaries are inserted before the manual changes loop
+                    } else if (dictionaryChange instanceof IdRemoval idRemoval) {
+                        return pendingDictionaryEntryFactory.buildRemoveFromDictionary(idRemoval, entityLogEntry);
+                    } else if (dictionaryChange instanceof ManualResizeRedaction manualResizeRedaction) {
+                        return pendingDictionaryEntryFactory.buildResizeWithDictionary(manualResizeRedaction, entityLogEntry);
+                    } else if (dictionaryChange instanceof ManualRecategorization manualRecategorization) {
+                        return pendingDictionaryEntryFactory.buildRecategorizeWithDictionary(manualRecategorization, entityLogEntry);
+                    } else {
+                        throw new IllegalArgumentException(String.format("Manual change of type %s has no defined dictionary action!", dictionaryChange.getClass()));
+                    }
+                })
+                .filter(Objects::nonNull)
+                .toList();
+    }
+
+
+    private List<EntityLogEntry> mergeLocalManualChangesAndReturnNonMergeableAsPending(DossierEntity dossier,
+                                                                                       Map<String, List<BaseAnnotation>> allManualChanges,
+                                                                                       EntityLogEntry entityLogEntry,
+                                                                                       int analysisNumber) {
+
+        return allManualChanges.getOrDefault(entityLogEntry.getId(), Collections.emptyList())
+                .stream()
+                .filter(BaseAnnotation::isLocal)
+                .sorted(Comparator.comparing(BaseAnnotation::getRequestDate))
+                .map(localChange -> {
+                    if (localChange instanceof IdRemoval idRemoval) {
+                        mergeIdToRemove(idRemoval, entityLogEntry, analysisNumber);
+                        return null;
+                    } else if (localChange instanceof ManualResizeRedaction manualResizeRedaction) {
+                        mergeResizeRedaction(manualResizeRedaction, entityLogEntry, analysisNumber);
+                        return null;
+                    } else if (localChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
+                        mergeLegalBasisChange(manualLegalBasisChange, entityLogEntry, analysisNumber);
+                        return null;
+                    } else if (localChange instanceof ManualRecategorization manualRecategorization) {
+                        return mergeRecategorization(manualRecategorization, entityLogEntry, dossier, analysisNumber);
+                    } else if (localChange instanceof ManualForceRedaction manualForceRedaction) {
+                        mergeForceRedaction(manualForceRedaction, entityLogEntry, analysisNumber);
+                        return null;
+                    } else {
+                        return null;
+                    }
+                })
+                .filter(Objects::nonNull)
+                .toList();
+    }
+
+
     private Optional<EntityLogEntry> mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) {
 
         if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) {
@@ -109,7 +239,10 @@ public class EntityLogMergeService {
         if (isFalsePositive(manualRedactionEntry)) {
             var matchingEntities = entityLog.getEntityLogEntry()
                     .stream()
-                    .filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0)))
+                    .filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions()
+                                                                    .get(0),
+                                                            entityLogEntry.getPositions()
+                                                                    .get(0)))
                     .toList();
             matchingEntities.forEach(matchingEntity -> mergeFalsePositive(entityLog, matchingEntity));
             return Optional.empty();
@@ -117,12 +250,12 @@ public class EntityLogMergeService {
 
         List<ManualChange> manualChanges = new ArrayList<>();
         manualChanges.add(ManualChange.builder()
-                .manualRedactionType(calculateManualRedactionType(manualRedactionEntry))
+                                  .manualRedactionType(ManualRedactionType.ADD)
-                .requestedDate(manualRedactionEntry.getRequestDate())
+                                  .requestedDate(manualRedactionEntry.getRequestDate())
-                .processedDate(null)
+                                  .processedDate(null)
-                .userId(manualRedactionEntry.getUser())
+                                  .userId(manualRedactionEntry.getUser())
-                .propertyChanges(Map.of("value", manualRedactionEntry.getValue()))
+                                  .propertyChanges(Map.of("value", manualRedactionEntry.getValue()))
-                .build());
+                                  .build());
 
         List<Change> changes = new ArrayList<>();
         changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED).build());
@@ -149,9 +282,13 @@ public class EntityLogMergeService {
                 .excluded(false)
                 .changes(changes)
                 .manualChanges(manualChanges)
-                .engines(new HashSet<>())
+                .engines(new HashSet<>(List.of(Engine.MANUAL)))
                 .reference(new HashSet<>())
                 .importedRedactionIntersections(new HashSet<>())
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .startOffset(-1)
+                .endOffset(-1)
                 .build();
 
         entityLog.getEntityLogEntry().add(entityLogEntry);
@@ -178,67 +315,71 @@ public class EntityLogMergeService {
     }
 
 
-    private void mergeIdsToRemove(IdRemoval idRemoval, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
+    private void mergeIdToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) {
 
-        var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
+        entityLogEntry.setState(EntryState.IGNORED);
-        entity.ifPresent(entityLogEntry -> {
+        entityLogEntry.getEngines().add(Engine.MANUAL);
-            entityLogEntry.setState(EntryState.IGNORED);
+        addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
-            addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
+        entityLogEntry.getManualChanges()
-            entityLogEntry.getManualChanges()
+                .add(ManualChange.builder()
-                    .add(ManualChange.builder()
+                             .manualRedactionType(ManualRedactionType.REMOVE)
-                            .manualRedactionType(idRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY)
+                             .requestedDate(idRemoval.getRequestDate())
-                            .requestedDate(idRemoval.getRequestDate())
+                             .processedDate(null)
-                            .processedDate(null)
+                             .userId(idRemoval.getUser())
-                            .userId(idRemoval.getUser())
+                             .propertyChanges(Collections.emptyMap())
-                            .build());
+                             .build());
-        });
     }
 
 
-    private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
+    private void mergeResizeRedaction(ManualResizeRedaction manualResizeRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
 
-        var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
+        entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
-        entity.ifPresent(entityLogEntry -> {
+        entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
-            entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
+        entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
-            entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
+        entityLogEntry.setValue(manualResizeRedaction.getValue());
-            entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
+        entityLogEntry.getEngines().add(Engine.MANUAL);
-            entityLogEntry.setValue(manualResizeRedaction.getValue());
+        addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualResizeRedaction.getRequestDate());
-            addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualResizeRedaction.getRequestDate());
+        ManualChange.ManualChangeBuilder manualChange = ManualChange.builder()
-            ManualChange.ManualChangeBuilder manualChange = ManualChange.builder()
+                .manualRedactionType(ManualRedactionType.RESIZE)
-                    .manualRedactionType(manualResizeRedaction.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE)
+                .requestedDate(manualResizeRedaction.getRequestDate())
-                    .requestedDate(manualResizeRedaction.getRequestDate())
+                .processedDate(null)
-                    .processedDate(null)
+                .userId(manualResizeRedaction.getUser());
-                    .userId(manualResizeRedaction.getUser());
+        if (!Strings.isNullOrEmpty(manualResizeRedaction.getValue())) {
-            if (!Strings.isNullOrEmpty(manualResizeRedaction.getValue())) {
+            manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue()));
-                manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue()));
+        }
-            }
+        entityLogEntry.getManualChanges().add(manualChange.build());
-            entityLogEntry.getManualChanges().add(manualChange.build());
-        });
     }
 
 
-    private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
+    private boolean isDuplicatedByAny(EntityLogEntry entry, Map<String, EntityLogEntry> addedLocalManualEntryIds) {
-
-        var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
-        entity.ifPresent(entityLogEntry -> {
-            entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
-            entityLogEntry.setSection(manualLegalBasisChange.getSection());
-            entityLogEntry.setValue(manualLegalBasisChange.getValue());
-            addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
-            Map<String, String> propertyChanges = getPropertyChanges(manualLegalBasisChange);
-            entityLogEntry.getManualChanges()
-                    .add(ManualChange.builder()
-                            .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
-                            .requestedDate(manualLegalBasisChange.getRequestDate())
-                            .processedDate(null)
-                            .propertyChanges(propertyChanges)
-                            .userId(manualLegalBasisChange.getUser())
-                            .build());
-        });
 
+        return !entry.getEngines().contains(Engine.MANUAL) && addedLocalManualEntryIds.containsKey(entry.getId()) && !addedLocalManualEntryIds.get(entry.getId()).equals(entry);
     }
 
 
+    @Deprecated(forRemoval = true)
+    private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange,
+                                       EntityLogEntry entityLogEntry,
+                                       int analysisNumber) {
+
+        entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
+        entityLogEntry.setSection(manualLegalBasisChange.getSection());
+        entityLogEntry.setValue(manualLegalBasisChange.getValue());
+        entityLogEntry.getEngines().add(Engine.MANUAL);
+        addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
+        Map<String, String> propertyChanges = getPropertyChanges(manualLegalBasisChange);
+        entityLogEntry.getManualChanges()
+                .add(ManualChange.builder()
+                             .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
+                             .requestedDate(manualLegalBasisChange.getRequestDate())
+                             .processedDate(null)
+                             .propertyChanges(propertyChanges)
+                             .userId(manualLegalBasisChange.getUser())
+                             .build());
+    }
+
+
+    @Deprecated(forRemoval = true)
     private Map<String, String> getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) {
 
         Map<String, String> propertyChanges = new HashMap<>();
@@ -255,44 +396,89 @@ public class EntityLogMergeService {
     }
 
 
-    private void mergeRecategorizations(ManualRecategorization recategorization, List<EntityLogEntry> entityLogEntries, DossierEntity dossier, int analysisNumber) {
+    private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization,
+                                                 EntityLogEntry entityLogEntry,
+                                                 DossierEntity dossier,
+                                                 int analysisNumber) {
+
+        if (entityLogEntry.getEntryType().equals(EntryType.IMAGE) || entityLogEntry.getEntryType().equals(EntryType.IMAGE_HINT)) {
+            return pendingDictionaryEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
+        }
 
         boolean isHint = isHint(recategorization.getType(), dossier);
-        var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
+
-        entity.ifPresent(entityLogEntry -> {
+        if (!Strings.isNullOrEmpty(recategorization.getType())) {
             entityLogEntry.setType(recategorization.getType());
-            entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType()));
+        }
+
+        entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType()));
+
+        entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); // TODO: only set applied if legalBasis is set by recategorization
+
+        entityLogEntry.getEngines().add(Engine.MANUAL);
+
+        if (!Strings.isNullOrEmpty(recategorization.getLegalBasis())) {
+            entityLogEntry.setLegalBasis(recategorization.getLegalBasis());
             entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
-            addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate());
+        } else {
-            entityLogEntry.getManualChanges()
+            entityLogEntry.setState(EntryState.SKIPPED);
-                    .add(ManualChange.builder()
+        }
-                            .manualRedactionType(ManualRedactionType.RECATEGORIZE)
+
-                            .requestedDate(recategorization.getRequestDate())
+        if (!Strings.isNullOrEmpty(recategorization.getSection())) {
-                            .processedDate(recategorization.getProcessedDate())
+            entityLogEntry.setSection(recategorization.getSection());
-                            .userId(recategorization.getUser())
+        }
-                            .propertyChanges(Map.of("type", recategorization.getType()))
+
-                            .build());
+        if (!Strings.isNullOrEmpty(recategorization.getValue())) {
-        });
+            entityLogEntry.setValue(recategorization.getValue());
+        }
+
+        addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate());
+        entityLogEntry.getManualChanges()
+                .add(ManualChange.builder()
+                             .manualRedactionType(ManualRedactionType.RECATEGORIZE)
+                             .requestedDate(recategorization.getRequestDate())
+                             .processedDate(recategorization.getProcessedDate())
+                             .userId(recategorization.getUser())
+                             .propertyChanges(getPropertyChanges(recategorization))
+                             .build());
+        return null;
     }
 
 
-    private void mergeForceRedactions(ManualForceRedaction forceRedaction, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
+    public static Map<String, String> getPropertyChanges(ManualRecategorization recategorization) {
 
-        var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
+        Map<String, String> propertyChanges = new HashMap<>();
-        entity.ifPresent(entityLogEntry -> {
+        if (!Strings.isNullOrEmpty(recategorization.getType())) {
-            entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
+            propertyChanges.put("type", recategorization.getType());
-            entityLogEntry.setState(entityLogEntry.getEntryType().equals(EntryType.HINT) ? EntryState.SKIPPED : EntryState.APPLIED);
+        }
-            addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
+        if (!Strings.isNullOrEmpty(recategorization.getLegalBasis())) {
-            var forceRedactManualChange = ManualChange.builder()
+            propertyChanges.put("legalBasis", recategorization.getLegalBasis());
-                    .manualRedactionType(entityLogEntry.getEntryType().equals(EntryType.HINT) ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT)
+        }
-                    .requestedDate(forceRedaction.getRequestDate())
+        if (!Strings.isNullOrEmpty(recategorization.getSection())) {
-                    .processedDate(forceRedaction.getProcessedDate())
+            propertyChanges.put("section", recategorization.getSection());
-                    .userId(forceRedaction.getUser());
+        }
-            if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
+        if (!Strings.isNullOrEmpty(recategorization.getValue())) {
-                forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
+            propertyChanges.put("value", recategorization.getValue());
-            }
+        }
-            entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
+        return propertyChanges;
-        });
+    }
+
+
+    private void mergeForceRedaction(ManualForceRedaction forceRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
+
+        entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
+        entityLogEntry.setState(entityLogEntry.getEntryType().equals(EntryType.HINT) ? EntryState.SKIPPED : EntryState.APPLIED);
+        entityLogEntry.getEngines().add(Engine.MANUAL);
+        addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
+        var forceRedactManualChange = ManualChange.builder()
+                .manualRedactionType(ManualRedactionType.FORCE)
+                .requestedDate(forceRedaction.getRequestDate())
+                .processedDate(forceRedaction.getProcessedDate())
+                .userId(forceRedaction.getUser());
+        if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
+            forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
+        }
+        entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
     }
 
 
@@ -306,15 +492,6 @@ public class EntityLogMergeService {
     }
 
 
-    private ManualRedactionType calculateManualRedactionType(ManualRedactionEntry manualRedactionEntry) {
-
-        if (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) {
-            return ManualRedactionType.ADD_TO_DICTIONARY;
-        }
-        return ManualRedactionType.ADD_LOCALLY;
-    }
-
-
     private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
 
         if (!changes.isEmpty()) {
@@ -331,7 +508,10 @@ public class EntityLogMergeService {
         TypeEntity typeEntity = dictionaryPersistenceService.getType(typeId);
 
         if (typeEntity == null) {
-            var optionalType = dictionaryPersistenceService.getAllTypes(false).stream().filter(typeEntity1 -> typeEntity1.getType().equals(type)).findFirst();
+            var optionalType = dictionaryPersistenceService.getAllTypes(false)
+                    .stream()
+                    .filter(typeEntity1 -> typeEntity1.getType().equals(type))
+                    .findFirst();
             if (optionalType.isPresent()) {
                 typeEntity = optionalType.get();
             } else {
@@ -345,7 +525,10 @@ public class EntityLogMergeService {
 
     private boolean equalPosition(Rectangle position1, Position position2) {
 
-        return position1.getTopLeftX() == position2.x() && position1.getTopLeftY() == position2.y() && position1.getWidth() == position2.w() && position1.getHeight() == position2.h();
+        return position1.getTopLeftX() == position2.x()
+               && position1.getTopLeftY() == position2.y()
+               && position1.getWidth() == position2.w()
+               && position1.getHeight() == position2.h();
     }
 
 
@@ -357,17 +540,6 @@ public class EntityLogMergeService {
     }
 
 
-    private List<BaseAnnotation> allManualChanges(ManualRedactions manualRedactions) {
-
-        return Stream.of(manualRedactions.getEntriesToAdd(),
-                manualRedactions.getForceRedactions(),
-                manualRedactions.getResizeRedactions(),
-                manualRedactions.getRecategorizations(),
-                manualRedactions.getIdsToRemove(),
-                manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
-    }
-
-
     public void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) {
 
         var fileEntity = fileStatusPersistenceService.getStatus(fileId);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java

@@ -8,9 +8,9 @@ import java.util.List;
 import java.util.Map;
 
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@@ -24,10 +24,10 @@ import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@Slf4j
 public class EntityLogService {
 
     FileManagementStorageService fileManagementStorageService;
@@ -64,18 +64,23 @@ public class EntityLogService {
 
         if (includeUnprocessed) {
             DossierEntity dossier = dossierService.getDossierById(dossierId);
-            ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true);
+            ManualRedactions unprocessedManualRedactions = manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.unprocessedOnly());
-            entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier);
+            entityLog = entityLogMergeService.mergeEntityLog(unprocessedManualRedactions, entityLog, dossier);
         }
 
         if (fileStatus.getExcludedPages() != null && !fileStatus.getExcludedPages().isEmpty()) {
             entityLog.getEntityLogEntry()
-                    .removeIf(entry -> entry.getPositions().stream().anyMatch(position -> fileStatus.getExcludedPages().contains(position.getPageNumber())) //
+                    .removeIf(entry -> entry.getPositions()
-                            && entry.getManualChanges().stream().noneMatch(m -> m.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY)));
+                                               .stream()
+                                               .anyMatch(position -> fileStatus.getExcludedPages().contains(position.getPageNumber())) //
+                                       && entry.getManualChanges()
+                                               .stream()
+                                               .noneMatch(m -> m.getManualRedactionType().equals(ManualRedactionType.ADD)));
         }
 
         Map<String, Integer> commentCountPerAnnotationId = commentService.getCommentCounts(fileId);
-        entityLog.getEntityLogEntry().forEach(entityLogEntry -> entityLogEntry.setNumberOfComments(commentCountPerAnnotationId.getOrDefault(entityLogEntry.getId(), 0)));
+        entityLog.getEntityLogEntry()
+                .forEach(entityLogEntry -> entityLogEntry.setNumberOfComments(commentCountPerAnnotationId.getOrDefault(entityLogEntry.getId(), 0)));
 
         return entityLog;
     }
@@ -102,7 +107,7 @@ public class EntityLogService {
             }
             for (var manualChange : redactionLogEntry.getManualChanges()) {
                 if (manualChange.getProcessedDate() != null && manualChange.getProcessedDate().isAfter(filteredEntityLogRequest.getSpecifiedDate()) || //
-                        manualChange.getRequestedDate() != null && manualChange.getRequestedDate().isAfter(filteredEntityLogRequest.getSpecifiedDate())) {
+                    manualChange.getRequestedDate() != null && manualChange.getRequestedDate().isAfter(filteredEntityLogRequest.getSpecifiedDate())) {
                     isAfterSpecifiedDate = true;
                     break;
                 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java

@@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
@@ -114,42 +115,44 @@ public class FileService {
 
     public void softDeleteFile(String dossierId, String fileId, OffsetDateTime softDeletedTime) {
 
-        forceRedactionPersistenceService.findForceRedactions(fileId, false)
+        ManualChangesQueryOptions options = ManualChangesQueryOptions.allWithoutDeleted();
+
+        forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     forceRedactionPersistenceService.softDelete(fileId, annotation.getId().getAnnotationId(), softDeletedTime);
                     commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), false)
                             .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime));
                 });
 
-        removeRedactionPersistenceService.findRemoveRedactions(fileId, false)
+        removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     removeRedactionPersistenceService.softDelete(fileId, annotation.getId().getAnnotationId(), softDeletedTime);
                     commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), false)
                             .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime));
                 });
 
-        addRedactionPersistenceService.findAddRedactions(fileId, false)
+        addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     addRedactionPersistenceService.softDelete(fileId, annotation.getId().getAnnotationId(), softDeletedTime);
                     commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), false)
                             .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime));
                 });
 
-        recategorizationPersistenceService.findRecategorizations(fileId, false)
+        recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(recatigorization -> {
                     recategorizationPersistenceService.softDelete(fileId, recatigorization.getId().getAnnotationId(), softDeletedTime);
                     commentPersistenceService.findCommentsByAnnotationId(fileId, recatigorization.getId().getAnnotationId(), false)
                             .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime));
                 });
 
-        resizeRedactionPersistenceService.findResizeRedactions(fileId, false)
+        resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     resizeRedactionPersistenceService.softDelete(fileId, annotation.getId().getAnnotationId(), softDeletedTime);
                     commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), false)
                             .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime));
                 });
 
-        legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false)
+        legalBasisChangePersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(legalBasisChange -> {
                     legalBasisChangePersistenceService.softDelete(fileId, legalBasisChange.getId().getAnnotationId(), softDeletedTime);
                     commentPersistenceService.findCommentsByAnnotationId(fileId, legalBasisChange.getId().getAnnotationId(), false)
@@ -187,35 +190,37 @@ public class FileService {
                     }
                 });
 
-        forceRedactionPersistenceService.findForceRedactions(fileId, true)
+        ManualChangesQueryOptions options = ManualChangesQueryOptions.all();
+
+        forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     forceRedactionPersistenceService.hardDelete(fileId, annotation.getId().getAnnotationId());
                     commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true)
                             .forEach(comment -> commentPersistenceService.hardDelete(comment.getId()));
                 });
 
-        removeRedactionPersistenceService.findRemoveRedactions(fileId, true)
+        removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     removeRedactionPersistenceService.hardDelete(fileId, annotation.getId().getAnnotationId());
                     commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true)
                             .forEach(comment -> commentPersistenceService.hardDelete(comment.getId()));
                 });
 
-        addRedactionPersistenceService.findAddRedactions(fileId, true)
+        addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     addRedactionPersistenceService.hardDelete(fileId, annotation.getId().getAnnotationId());
                     commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true)
                             .forEach(comment -> commentPersistenceService.hardDelete(comment.getId()));
                 });
 
-        recategorizationPersistenceService.findRecategorizations(fileId, true)
+        recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(recatigorization -> {
                     recategorizationPersistenceService.hardDelete(fileId, recatigorization.getId().getAnnotationId());
                     commentPersistenceService.findCommentsByAnnotationId(fileId, recatigorization.getId().getAnnotationId(), true)
                             .forEach(comment -> commentPersistenceService.hardDelete(comment.getId()));
                 });
 
-        resizeRedactionPersistenceService.findResizeRedactions(fileId, true)
+        resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     resizeRedactionPersistenceService.hardDelete(fileId, annotation.getId().getAnnotationId());
                     commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true)
@@ -246,7 +251,8 @@ public class FileService {
 
     public void undeleteFile(String dossierTemplateId, String dossierId, String fileId, OffsetDateTime softDeletedTime) {
 
-        forceRedactionPersistenceService.findForceRedactions(fileId, true)
+
					
+        forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     if (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime().isAfter(softDeletedTime)) {
                         forceRedactionPersistenceService.undelete(fileId, annotation.getId().getAnnotationId());
@@ -259,7 +265,7 @@ public class FileService {
                     }
                 });
 
-        removeRedactionPersistenceService.findRemoveRedactions(fileId, true)
+        removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     if (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime().isAfter(softDeletedTime)) {
                         removeRedactionPersistenceService.undelete(fileId, annotation.getId().getAnnotationId());
@@ -272,7 +278,7 @@ public class FileService {
                     }
                 });
 
-        addRedactionPersistenceService.findAddRedactions(fileId, true)
+        addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     if (annotation != null && annotation.getSoftDeletedTime() != null && (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime()
                             .isAfter(softDeletedTime))) {
@@ -286,7 +292,7 @@ public class FileService {
                     }
                 });
 
-        recategorizationPersistenceService.findRecategorizations(fileId, true)
+        recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(recatigorization -> {
                     if (recatigorization.getSoftDeletedTime().equals(softDeletedTime) || recatigorization.getSoftDeletedTime().isAfter(softDeletedTime)) {
                         recategorizationPersistenceService.undelete(fileId, recatigorization.getId().getAnnotationId());
@@ -299,7 +305,7 @@ public class FileService {
                     }
                 });
 
-        resizeRedactionPersistenceService.findResizeRedactions(fileId, true)
+        resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     if (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime().isAfter(softDeletedTime)) {
                         resizeRedactionPersistenceService.undelete(fileId, annotation.getId().getAnnotationId());
@@ -312,7 +318,7 @@ public class FileService {
                     }
                 });
 
-        legalBasisChangePersistenceService.findLegalBasisChanges(fileId, true)
+        legalBasisChangePersistenceService.findEntriesByFileIdAndOptions(fileId, options)
                 .forEach(annotation -> {
                     if (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime().isAfter(softDeletedTime)) {
                         legalBasisChangePersistenceService.undelete(fileId, annotation.getId().getAnnotationId());

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
 import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceRequest;
 import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.model.NerServiceRequest;
 import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse;
 import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest;
@@ -226,7 +227,7 @@ public class FileStatusService {
                 .analysisNumber(fileModel.getNumberOfAnalyses() + 1)
                 .sectionsToReanalyse(sectionsToReanalyse)
                 .fileId(fileId)
-                .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
+                .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
                 .dossierTemplateId(dossier.getDossierTemplateId())
                 .lastProcessed(fileModel.getLastProcessed())
                 .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
@@ -695,27 +696,28 @@ public class FileStatusService {
         comments.forEach((key, value) -> value.forEach(comment -> commentPersistenceService.softDelete(comment.getId(), now)));
 
         // wipe force redactions
-        var forceRedactions = forceRedactionPersistenceService.findForceRedactions(fileId, false);
+        ManualChangesQueryOptions options = ManualChangesQueryOptions.allWithoutDeleted();
+        var forceRedactions = forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options);
         forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
 
         // wipe add manual redactions
-        var addRedactions = addRedactionPersistenceService.findAddRedactions(fileId, false);
+        var addRedactions = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options);
         addRedactions.forEach(f -> addRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
 
         // wipe removeRedactions
-        var removeRedactions = removeRedactionPersistenceService.findRemoveRedactions(fileId, false);
+        var removeRedactions = removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options);
         removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
 
         // wipe image recat
-        var imageRecategorizations = recategorizationPersistenceService.findRecategorizations(fileId, false);
+        var imageRecategorizations = recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options);
         imageRecategorizations.forEach(f -> recategorizationPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));// wipe image recat
 
         // wipe resize redactions
-        var resizeRedactions = resizeRedactionPersistenceService.findResizeRedactions(fileId, false);
+        var resizeRedactions = resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options);
         resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
 
         // wipe legal basis changes
-        var legalBasisChanges = legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false);
+        var legalBasisChanges = legalBasisChangePersistenceService.findEntriesByFileIdAndOptions(fileId, options);
         legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
 
         fileStatusPersistenceService.updateHasComments(fileId, false);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/IndexingService.java

@@ -9,8 +9,6 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@@ -25,7 +23,6 @@ import lombok.RequiredArgsConstructor;
 public class IndexingService {
 
     private final RabbitTemplate rabbitTemplate;
-    private final ObjectMapper objectMapper;
     private final DossierService dossierService;
     private final FileStatusPersistenceService fileStatusPersistenceService;
 
@@ -39,13 +36,14 @@ public class IndexingService {
             if (dossierId == null) {
                 List<DossierEntity> dossiers = dossierService.getAllDossiers();
                 for (DossierEntity dossier : dossiers) {
-                    if (dossier.getSoftDeletedTime() == null || dossier.getArchivedTime() != null) {
+                    if (dossier.getHardDeletedTime() != null || dossier.getArchivedTime() != null) {
-                        reindexDossierIds.add(new ImmutablePair<>(dossier.getDossierTemplateId(), dossier.getId()));
+                        continue;
                     }
+                    reindexDossierIds.add(new ImmutablePair<>(dossier.getDossierTemplateId(), dossier.getId()));
                 }
             } else {
                 DossierEntity dossier = dossierService.getDossierById(dossierId);
-                if (dossier.getSoftDeletedTime() == null || dossier.getArchivedTime() != null) {
+                if (dossier.getHardDeletedTime() == null && dossier.getArchivedTime() == null) {
                     reindexDossierIds.add(new ImmutablePair<>(dossier.getDossierTemplateId(), dossier.getId()));
                 }
             }
@@ -53,7 +51,7 @@ public class IndexingService {
             for (Pair<String, String> reindexDossierId : reindexDossierIds) {
                 List<FileEntity> fileStatuses = fileStatusPersistenceService.getStatusesForDossier(reindexDossierId.getRight());
                 for (FileEntity fileStatus : fileStatuses) {
-                    if (fileStatus.isSoftOrHardDeleted()) {
+                    if (fileStatus.getHardDeletedTime() != null) {
                         continue;
                     }
                     if (fileIds != null && !fileIds.isEmpty() && !fileIds.contains(fileStatus.getId())) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/KeyCloakUserSyncService.java

@@ -53,7 +53,7 @@ public class KeyCloakUserSyncService {
             // remove KC users, what's left is users that are in redaction but no longer in KC
             redactionObjectsUserIds.removeAll(allUserIds);
 
-            log.info("Performing user sync/cleanup for ids: {}", redactionObjectsUserIds);
+            log.debug("Performing user sync/cleanup for ids: {}", redactionObjectsUserIds);
 
             redactionObjectsUserIds.forEach(removedUser -> alLDossiers.forEach(dossier -> this.userService.updateDossierUsers(removedUser,
                     UserService.UserRemovalModel.PERMANENT,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java

@@ -87,7 +87,9 @@ public class ReanalysisRequiredStatusService {
             return new AnalysisRequiredResult(false, true);
         }
 
-        if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.PRE_PROCESSED.equals(fileStatus.getProcessingStatus()) || ignoreProcessingStates) {
+        if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus())
+            || ProcessingStatus.PRE_PROCESSED.equals(fileStatus.getProcessingStatus())
+            || ignoreProcessingStates) {
 
             switch (fileStatus.getWorkflowStatus()) {
                 case NEW:
@@ -130,7 +132,8 @@ public class ReanalysisRequiredStatusService {
         var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches;
         if (reanalysisRequired || fullAnalysisRequired) {
             log.info(
-                    "For file: {} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}",
+                    "For file: {}-{} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}",
+                    fileStatus.getId(),
                     fileStatus.getFilename(),
                     fileStatus.getRulesVersion(),
                     dossierTemplateVersions.getOrDefault(RULES, -1L),

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/export/DossierTemplateExportService.java

@@ -1,5 +1,20 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.export;
 
+import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
@@ -13,7 +28,16 @@ import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJ
 import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -32,19 +56,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
+
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import java.io.*;
-import java.nio.charset.StandardCharsets;
-import java.util.Iterator;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;
 
 @Slf4j
 @Service
@@ -77,6 +92,8 @@ public class DossierTemplateExportService {
 
         var mimeType = "application/zip";
 
+        dossierTemplatePersistenceService.validateDossierTemplateForDuplicateRanks(request.getDossierTemplateId());
+
         String downloadFilename = request.getDossierTemplateId() + ".zip";
         String storageId = StorageIdUtils.getStorageId(request.getUserId(), request.getDossierTemplateId());
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java

@@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
 import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
@@ -34,6 +35,7 @@ public class AutomaticAnalysisJob implements Job {
     private final FileStatusService fileStatusService;
     private final TenantProvider tenantProvider;
     private final ObservationRegistry observationRegistry;
+    private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
 
     @Setter
     private boolean schedulingStopped;
@@ -49,49 +51,61 @@ public class AutomaticAnalysisJob implements Job {
             return;
         }
 
-        tenantProvider.getTenants().forEach(tenant -> {
+        tenantProvider.getTenants()
+                .forEach(tenant -> {
 
-            if (!TenantUtils.isTenantReadyForPersistence(tenant) || stoppedTenants.contains(tenant.getTenantId())) {
+                    if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
-                return;
+                        log.info("[Tenant:{}] Skipping scheduling since tenant is not ready.", tenant.getTenantId());
-            }
+                        return;
-
-            TenantContext.setTenantId(tenant.getTenantId());
-
-            var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
-            if (redactionQueueInfo != null) {
-                log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
-                        tenant.getTenantId(),
-                        MessagingConfiguration.REDACTION_QUEUE,
-                        redactionQueueInfo.getMessageCount(),
-                        redactionQueueInfo.getConsumerCount());
-                // only 1 file in queue
-                var consumerCount = redactionQueueInfo.getConsumerCount();
-                if (redactionQueueInfo.getMessageCount() <= consumerCount * 5) {
-                    // queue up 5 files
-                    var allStatuses = getAllRelevantStatuses();
-
-                    allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated));
-
-                    var allStatusesIterator = allStatuses.iterator();
-                    log.debug("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size());
-
-                    var queuedFiles = 0;
-
-                    while (queuedFiles < (consumerCount * 5) && allStatusesIterator.hasNext()) {
-                        var next = allStatusesIterator.next();
-                        // in case the file doesn't have numberOfPages set, we assume an average.
-
-                        reanalyseFile(next);
-
-                        queuedFiles++;
                     }
-                }
 
-            } else {
+                    if (stoppedTenants.contains(tenant.getTenantId())) {
-                log.info("[Tenant:{}] Failed to obtain queue info for queue: {}", TenantContext.getTenantId(), MessagingConfiguration.REDACTION_QUEUE);
+                        log.info("[Tenant:{}] Skipping scheduling as automatic reanalysis is disabled for tenant.", tenant.getTenantId());
-            }
+                        return;
+                    }
 
-        });
+                    TenantContext.setTenantId(tenant.getTenantId());
+
+                    if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) {
+                        log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId());
+                        return;
+                    }
+
+                    var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
+                    if (redactionQueueInfo != null) {
+                        log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
+                                  tenant.getTenantId(),
+                                  MessagingConfiguration.REDACTION_QUEUE,
+                                  redactionQueueInfo.getMessageCount(),
+                                  redactionQueueInfo.getConsumerCount());
+                        // only 1 file in queue
+                        var consumerCount = redactionQueueInfo.getConsumerCount();
+                        if (redactionQueueInfo.getMessageCount() <= consumerCount * 5) {
+                            // queue up 5 files
+                            var allStatuses = getAllRelevantStatuses();
+
+                            allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated));
+
+                            var allStatusesIterator = allStatuses.iterator();
+                            log.debug("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size());
+
+                            var queuedFiles = 0;
+
+                            while (queuedFiles < (consumerCount * 5) && allStatusesIterator.hasNext()) {
+                                var next = allStatusesIterator.next();
+                                // in case the file doesn't have numberOfPages set, we assume an average.
+
+                                reanalyseFile(next);
+
+                                queuedFiles++;
+                            }
+                        }
+
+                    } else {
+                        log.info("[Tenant:{}] Failed to obtain queue info for queue: {}", TenantContext.getTenantId(), MessagingConfiguration.REDACTION_QUEUE);
+                    }
+
+                });
     }
 
 
@@ -106,10 +120,10 @@ public class AutomaticAnalysisJob implements Job {
                 .observe(() -> {
 
                     if (file.isFullAnalysisRequired()) {
-                        log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getFilename());
+                        log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getId());
                         fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false);
                     } else if (file.isReanalysisRequired()) {
-                        log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getFilename());
+                        log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getId());
                         fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), false);
                     }
                 });
@@ -124,12 +138,14 @@ public class AutomaticAnalysisJob implements Job {
 
     public void stopForTenant(String tenantId) {
 
+        log.info("Stopping automatic analysis for tenant {}", tenantId);
         stoppedTenants.add(tenantId);
     }
 
 
     public void startForTenant(String tenantId) {
 
+        log.info("Starting automatic analysis for tenant {}", tenantId);
         stoppedTenants.remove(tenantId);
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionDictionaryUpdateHandler.java

@@ -52,7 +52,7 @@ public class ManualRedactionDictionaryUpdateHandler {
 
     public Set<String> handleAddToDictionaryAndReturnModifiedTypeIds(String fileId, String value, ManualRequestWithAddToDictionary manualRequestWithAddToDictionary) {
 
-        if (!manualRequestWithAddToDictionary.isAddToDictionary()) {
+        if (!manualRequestWithAddToDictionary.isAddToDictionary() || value == null) {
             return Collections.emptySet();
         }
         Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
@@ -99,7 +99,7 @@ public class ManualRedactionDictionaryUpdateHandler {
 
     public Set<String> handleRemoveFromDictionaryAndReturnModifiedTypeIds(String fileId, ManualRequestWithRemoveFromDictionary manualRequestWithRemoveFromDictionary) {
 
-        if (!manualRequestWithRemoveFromDictionary.isRemoveFromDictionary()) {
+        if (!manualRequestWithRemoveFromDictionary.isRemoveFromDictionary() || manualRequestWithRemoveFromDictionary.getValue() == null) {
             return Collections.emptySet();
         }
         String dossierId = manualRequestWithRemoveFromDictionary.getDossierId();
@@ -300,7 +300,8 @@ public class ManualRedactionDictionaryUpdateHandler {
         resizeRedactionsWithSameValue.forEach(resizeRedaction -> {
             var file = fileStatusPersistenceService.getStatus(resizeRedaction.getId().getFileId());
             var dossierForResizeRedaction = dossierPersistenceService.findByDossierId(file.getDossierId());
-            if (!file.getWorkflowStatus().equals(WorkflowStatus.APPROVED) && dossierTemplateId.equals(dossierForResizeRedaction.getDossierTemplateId())) {
+            if (!file.getWorkflowStatus().equals(WorkflowStatus.APPROVED) && dossierTemplateId.equals(dossierForResizeRedaction.getDossierTemplateId())
+            && (resizeRedaction.isAddToAllDossiers() || resizeRedaction.getUpdateDictionary())) {
                 resizeRedactionPersistenceService.hardDelete(resizeRedaction.getId().getFileId(), resizeRedaction.getId().getAnnotationId());
             }
         });

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java

@@ -5,13 +5,15 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ty
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.function.Consumer;
 
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
@@ -49,11 +51,11 @@ public class ManualRedactionMapper {
 
         return addRedactionRequests.stream()
                 .map(addRedactionRequest -> toAddRedactionRequest(dossierId, dossier.getDossierTemplateId(), addRedactionRequest))
-                .collect(Collectors.toList());
+                .toList();
     }
 
 
-    public AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) {
+    private static AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) {
 
         return AddRedactionRequest.builder()
                 .value(StringCleaningUtility.cleanString(addRedactionRequest.getValue()))
@@ -106,31 +108,64 @@ public class ManualRedactionMapper {
     }
 
 
-    public List<ForceRedactionRequest> toForceRedactionRequestList(Set<ForceRedactionRequestModel> forceRedactionRequests) {
+    public List<ForceRedactionRequest> toForceRedactionRequestList(String dossierId,
+                                                                   String fileId,
+                                                                   Set<ForceRedactionRequestModel> forceRedactionRequests,
+                                                                   Consumer<EntityLogEntry> entityLogEntryConsumer) {
 
-        return forceRedactionRequests.stream()
+        EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId, Collections.emptyList(), true);
-                .map(forceRedactionRequest -> ForceRedactionRequest.builder()
+        List<ForceRedactionRequest> requests = new ArrayList<>();
-                        .annotationId(forceRedactionRequest.getAnnotationId())
+
-                        .user(KeycloakSecurity.getUserId())
+        for (ForceRedactionRequestModel forceRedactionRequestModel : forceRedactionRequests) {
-                        .legalBasis(forceRedactionRequest.getLegalBasis())
+
-                        .comment(forceRedactionRequest.getComment())
+            EntityLogEntry entityLogEntry = getEntityLogEntry(entityLog, forceRedactionRequestModel.getAnnotationId());
-                        .build())
+            ForceRedactionRequest forceRedactionRequest = ForceRedactionRequest.builder()
-                .collect(Collectors.toList());
+                    .annotationId(forceRedactionRequestModel.getAnnotationId())
+                    .user(KeycloakSecurity.getUserId())
+                    .legalBasis(forceRedactionRequestModel.getLegalBasis())
+                    .comment(forceRedactionRequestModel.getComment())
+                    .build();
+
+            if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
+                entityLogEntryConsumer.accept(entityLogEntry);
+            }
+
+            requests.add(forceRedactionRequest);
+        }
+
+        return requests;
     }
 
 
-    public List<LegalBasisChangeRequest> toLegalBasisChangeRequestList(Set<LegalBasisChangeRequestModel> legalBasisChangeRequests) {
+    @Deprecated(forRemoval = true)
+    public List<LegalBasisChangeRequest> toLegalBasisChangeRequestList(String dossierId,
+                                                                       String fileId,
+                                                                       Set<LegalBasisChangeRequestModel> legalBasisChangeRequests,
+                                                                       Consumer<EntityLogEntry> entityLogEntryConsumer) {
 
-        return legalBasisChangeRequests.stream()
+        EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId, Collections.emptyList(), true);
-                .map(legalBasisChangeRequest -> LegalBasisChangeRequest.builder()
+        List<LegalBasisChangeRequest> requests = new ArrayList<>();
-                        .annotationId(legalBasisChangeRequest.getAnnotationId())
+
-                        .user(KeycloakSecurity.getUserId())
+        for (LegalBasisChangeRequestModel legalBasisChangeRequest : legalBasisChangeRequests) {
-                        .section(legalBasisChangeRequest.getSection())
+
-                        .legalBasis(legalBasisChangeRequest.getLegalBasis())
+            EntityLogEntry entityLogEntry = getEntityLogEntry(entityLog, legalBasisChangeRequest.getAnnotationId());
-                        .comment(legalBasisChangeRequest.getComment())
+            LegalBasisChangeRequest request = LegalBasisChangeRequest.builder()
-                        .value(legalBasisChangeRequest.getValue())
+                    .annotationId(legalBasisChangeRequest.getAnnotationId())
-                        .build())
+                    .user(KeycloakSecurity.getUserId())
-                .collect(Collectors.toList());
+                    .section(legalBasisChangeRequest.getSection())
+                    .legalBasis(legalBasisChangeRequest.getLegalBasis())
+                    .comment(legalBasisChangeRequest.getComment())
+                    .value(legalBasisChangeRequest.getValue())
+                    .build();
+
+            if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
+                entityLogEntryConsumer.accept(entityLogEntry);
+            }
+
+            requests.add(request);
+        }
+
+        return requests;
     }
 
 
@@ -138,13 +173,16 @@ public class ManualRedactionMapper {
                                                                        String fileId,
                                                                        String dossierTemplateId,
                                                                        Set<RecategorizationRequestModel> recategorizationRequests,
-                                                                       boolean includeUnprocessed) {
+                                                                       boolean includeUnprocessed,
+                                                                       Consumer<EntityLogEntry> entityLogEntryConsumer) {
 
         EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId, Collections.emptyList(), includeUnprocessed);
         List<RecategorizationRequest> requests = new ArrayList<>();
+
         for (RecategorizationRequestModel recategorizationRequest : recategorizationRequests) {
+
             EntityLogEntry entityLogEntry = getEntityLogEntry(entityLog, recategorizationRequest.getAnnotationId());
-            RecategorizationRequest build = RecategorizationRequest.builder()
+            RecategorizationRequest request = RecategorizationRequest.builder()
                     .annotationId(recategorizationRequest.getAnnotationId())
                     .user(KeycloakSecurity.getUserId())
                     .dossierTemplateId(dossierTemplateId)
@@ -153,34 +191,69 @@ public class ManualRedactionMapper {
                     .addToDictionary(recategorizationRequest.isAddToDictionary())
                     .addToAllDossiers(recategorizationRequest.isAddToAllDossiers())
                     .dictionaryEntryType(getDictionaryEntryType(entityLogEntry))
-                    .value(entityLogEntry.getValue())
+                    .value(recategorizationRequest.getValue())
                     .typeToRemove(entityLogEntry.getType())
                     .dossierTemplateTypeId(toTypeId(recategorizationRequest.getType(), dossierTemplateId))
+                    .legalBasis(Optional.ofNullable(recategorizationRequest.getLegalBasis())
+                                        .orElse(""))
+                    .section(recategorizationRequest.getSection())
                     .build();
-            requests.add(build);
+
+            if (!entityLogEntry.getEngines().contains(Engine.MANUAL)
+                && !recategorizationRequest.isAddToAllDossiers()
+                && !recategorizationRequest.isAddToDictionary()
+                && entryIsEntityType(entityLogEntry)) {
+                entityLogEntryConsumer.accept(entityLogEntry);
+            }
+
+            requests.add(request);
         }
 
         return requests;
     }
 
 
-    public List<ResizeRedactionRequest> toResizeRedactionRequestList(Set<ResizeRedactionRequestModel> resizeRedactionRequests) {
+    public List<ResizeRedactionRequest> toResizeRedactionRequestList(Set<ResizeRedactionRequestModel> resizeRedactionRequests,
+                                                                     EntityLog entityLog,
+                                                                     Consumer<EntityLogEntry> entityLogEntryConsumer) {
 
-        return resizeRedactionRequests.stream()
+        List<ResizeRedactionRequest> requests = new ArrayList<>();
-                .map(resizeRedactionRequest -> ResizeRedactionRequest.builder()
+
-                        .annotationId(resizeRedactionRequest.getAnnotationId())
+        for (ResizeRedactionRequestModel resizeRedactionRequest : resizeRedactionRequests) {
-                        .user(KeycloakSecurity.getUserId())
+
-                        .positions(resizeRedactionRequest.getPositions())
+            EntityLogEntry entityLogEntry = getEntityLogEntry(entityLog, resizeRedactionRequest.getAnnotationId());
-                        .value(resizeRedactionRequest.getValue() == null ? "" : StringCleaningUtility.cleanString(resizeRedactionRequest.getValue()))
+            ResizeRedactionRequest request = ResizeRedactionRequest.builder()
-                        .comment(resizeRedactionRequest.getComment())
+                    .annotationId(resizeRedactionRequest.getAnnotationId())
-                        .updateDictionary(resizeRedactionRequest.getUpdateDictionary())
+                    .user(KeycloakSecurity.getUserId())
-                        .addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers())
+                    .positions(resizeRedactionRequest.getPositions())
-                        .build())
+                    .value(resizeRedactionRequest.getValue() == null ? "" : StringCleaningUtility.cleanString(resizeRedactionRequest.getValue()))
-                .collect(Collectors.toList());
+                    .comment(resizeRedactionRequest.getComment())
+                    .updateDictionary(resizeRedactionRequest.getUpdateDictionary())
+                    .addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers())
+                    .build();
+
+            if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) {
+                entityLogEntryConsumer.accept(entityLogEntry);
+            }
+
+            requests.add(request);
+        }
+
+        return requests;
     }
 
 
-    private EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) {
+    private static boolean entryIsEntityType(EntityLogEntry entityLogEntry) {
+
+        return entityLogEntry.getEntryType().equals(EntryType.ENTITY) //
+               || entityLogEntry.getEntryType().equals(EntryType.HINT) //
+               || entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) //
+               || entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION) //
+               || entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE);
+    }
+
+
+    private static EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) {
 
         return entityLog.getEntityLogEntry()
                 .stream()
@@ -190,9 +263,9 @@ public class ManualRedactionMapper {
     }
 
 
-    private DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) {
+    public static DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) {
 
-        if (entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) && entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
+        if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) {
             return DictionaryEntryType.FALSE_RECOMMENDATION;
         } else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
             return DictionaryEntryType.FALSE_POSITIVE;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java

@@ -2,17 +2,16 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual
 
 import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;
 
+import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.springframework.dao.EmptyResultDataAccessException;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
@@ -20,8 +19,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RecategorizationPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.utils.ManualImageRecategorizationMapper;
+import com.iqser.red.service.persistence.management.v1.processor.utils.ManualRecategorizationMapper;
-import com.iqser.red.service.persistence.management.v1.processor.utils.ManualRedactionMapper;
+import com.iqser.red.service.persistence.management.v1.processor.utils.ManualRedactionEntryMapper;
 import com.iqser.red.service.persistence.management.v1.processor.utils.ManualResizeRedactionMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -49,13 +48,7 @@ public class ManualRedactionProviderService {
 
 
     @Transactional
-    public ManualRedactions getManualRedactions(String fileId) {
+    public ManualRedactions getManualRedactions(String fileId, ManualChangesQueryOptions options) {
-
-        return getManualRedactions(fileId, false);
-    }
-
-
-    public ManualRedactions getManualRedactions(String fileId, boolean unprocessed) {
 
         Set<ManualRedactionEntry> entriesToAdd;
         Set<IdRemoval> removals;
@@ -64,37 +57,50 @@ public class ManualRedactionProviderService {
         Set<ManualLegalBasisChange> legalBasisChanges;
         Set<ManualResizeRedaction> resizeRedactions;
 
-        if (unprocessed) {
+        if (!options.getExcludedClasses().contains(ManualRedactionEntry.class)) {
-            entriesToAdd = convertEntriesToAdd(addRedactionPersistenceService.findUnprocessedRedactions(fileId));
+            entriesToAdd = new HashSet<>(convert(addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options),
-            removals = convert(removeRedactionPersistenceService.findUnprocessedRemoveRedactions(fileId), IdRemoval.class);
+                                                 ManualRedactionEntry.class,
-            forceRedactions = convert(forceRedactionPersistenceService.findUnprocessedForceRedactions(fileId), ManualForceRedaction.class);
+                                                 new ManualRedactionEntryMapper()));
-            recategorizations = new HashSet<>(convert(recategorizationPersistenceService.findUnprocessedRecategorizations(fileId),
-                    ManualRecategorization.class,
-                    new ManualImageRecategorizationMapper()));
-            legalBasisChanges = convert(legalBasisChangePersistenceService.findUnprocessedLegalBasisChanges(fileId), ManualLegalBasisChange.class);
-            resizeRedactions = new HashSet<>(convert(resizeRedactionPersistenceService.findUnprocessedResizeRedactions(fileId),
-                    ManualResizeRedaction.class,
-                    new ManualResizeRedactionMapper()));
         } else {
-            entriesToAdd = convertEntriesToAdd(addRedactionPersistenceService.findAddRedactions(fileId, false));
+            entriesToAdd = Collections.emptySet();
-            removals = convert(removeRedactionPersistenceService.findRemoveRedactions(fileId, false), IdRemoval.class);
+        }
-            forceRedactions = convert(forceRedactionPersistenceService.findForceRedactions(fileId, false), ManualForceRedaction.class);
+
-            recategorizations = new HashSet<>(convert(recategorizationPersistenceService.findRecategorizations(fileId, false),
+        if (!options.getExcludedClasses().contains(ManualRecategorization.class)) {
-                    ManualRecategorization.class,
+            recategorizations = new HashSet<>(convert(recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options),
-                    new ManualImageRecategorizationMapper()));
+                                                      ManualRecategorization.class,
-            legalBasisChanges = convert(legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false), ManualLegalBasisChange.class);
+                                                      new ManualRecategorizationMapper()));
-            resizeRedactions = new HashSet<>(convert(resizeRedactionPersistenceService.findResizeRedactions(fileId, false),
+        } else {
-                    ManualResizeRedaction.class,
+            recategorizations = Collections.emptySet();
-                    new ManualResizeRedactionMapper()));
+        }
+
+        if (!options.getExcludedClasses().contains(ManualResizeRedaction.class)) {
+            resizeRedactions = new HashSet<>(convert(resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options),
+                                                     ManualResizeRedaction.class,
+                                                     new ManualResizeRedactionMapper()));
+        } else {
+            resizeRedactions = Collections.emptySet();
+        }
+
+        if (!options.getExcludedClasses().contains(IdRemoval.class)) {
+            removals = new HashSet<>(convert(removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options), IdRemoval.class));
+        } else {
+            removals = Collections.emptySet();
+        }
+
+        if (!options.getExcludedClasses().contains(ManualForceRedaction.class)) {
+            forceRedactions = new HashSet<>(convert(forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options), ManualForceRedaction.class));
+        } else {
+            forceRedactions = Collections.emptySet();
+        }
+
+        if (!options.getExcludedClasses().contains(ManualLegalBasisChange.class)) {
+            legalBasisChanges = new HashSet<>(convert(legalBasisChangePersistenceService.findEntriesByFileIdAndOptions(fileId, options), ManualLegalBasisChange.class));
+        } else {
+            legalBasisChanges = Collections.emptySet();
         }
 
         return new ManualRedactions(removals, entriesToAdd, forceRedactions, recategorizations, legalBasisChanges, resizeRedactions);
-    }
 
-
-    private Set<ManualRedactionEntry> convertEntriesToAdd(List<ManualRedactionEntryEntity> source) {
-
-        return source.stream().map(entry -> convert(entry, ManualRedactionEntry.class, new ManualRedactionMapper())).collect(Collectors.toSet());
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.manualredactions;
 
+import static com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionMapper.getDictionaryEntryType;
 import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert;
 
 import java.nio.charset.StandardCharsets;
@@ -8,6 +9,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
@@ -15,10 +17,13 @@ import org.springframework.transaction.annotation.Transactional;
 
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
 import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService;
 import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
@@ -31,21 +36,34 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RecategorizationPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.utils.ManualRedactionMapper;
+import com.iqser.red.service.persistence.management.v1.processor.utils.ManualRedactionEntryMapper;
 import com.iqser.red.service.persistence.management.v1.processor.utils.ManualResizeRedactionMapper;
+import com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ResizeRedactionRequestModel;
+import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
+import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
@@ -74,18 +92,21 @@ public class ManualRedactionService {
     ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler;
     EntityLogMergeService entityLogMergeService;
     FileStatusPersistenceService fileStatusPersistenceService;
+    ManualRedactionMapper manualRedactionMapper;
+
+    private static final int MAX_LEGAL_BASIS_LENGTH = 4000;
 
 
     @Transactional
     @Observed(name = "ManualRedactionService", contextualName = "add-manual-redaction")
-    public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, List<AddRedactionRequest> addRedactionRequests) {
+    public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, Set<AddRedactionRequestModel> addRedactionRequests, Dossier dossier) {
 
         var response = new ArrayList<ManualAddResponse>();
         List<ManualRedactionEntryEntity> manualRedactionEntryEntities = new ArrayList<>();
-
+        List<AddRedactionRequest> requests = manualRedactionMapper.toAddRedactionRequestList(dossierId, addRedactionRequests, dossier);
         var dossierEntity = dossierPersistenceService.getAndValidateDossier(dossierId);
 
-        for (AddRedactionRequest addRedactionRequest : addRedactionRequests) {
+        for (AddRedactionRequest addRedactionRequest : requests) {
             manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue());
             validatePositions(fileId, addRedactionRequest);
 
@@ -94,8 +115,8 @@ public class ManualRedactionService {
             manualRedactionEntryEntities.add(addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest));
 
             Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
-                    addRedactionRequest.getValue(),
+                                                                                                                                             addRedactionRequest.getValue(),
-                    addRedactionRequest);
+                                                                                                                                             addRedactionRequest);
 
             addRedactionPersistenceService.updateModifiedDictionaries(fileId, annotationId, !typeIdsOfModifiedDictionaries.isEmpty(), typeIdsOfModifiedDictionaries);
 
@@ -104,13 +125,13 @@ public class ManualRedactionService {
             response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
         }
 
-        manualRedactionEntryEntities = manualRedactionEntryEntities.stream()
+        var localManualRedactionEntryEntities = manualRedactionEntryEntities.stream()
-                .filter(manualRedactionEntry -> !manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
+                .filter(manualRedactionEntry -> !manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToAllDossiers())
                 .collect(Collectors.toList());
 
         FileModel fileStatus = fileStatusService.getStatus(fileId);
-        if (!manualRedactionEntryEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) {
+        if (!localManualRedactionEntryEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) {
-            ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(manualRedactionEntryEntities)).build();
+            ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(localManualRedactionEntryEntities)).build();
             entityLogMergeService.sendToAnalyseQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions);
         } else {
             reprocess(dossierId, fileId);
@@ -123,37 +144,45 @@ public class ManualRedactionService {
 
     private Set<ManualRedactionEntry> convertEntriesToAdd(List<ManualRedactionEntryEntity> source) {
 
-        return source.stream().map(entry -> convert(entry, ManualRedactionEntry.class, new ManualRedactionMapper())).collect(Collectors.toSet());
+        return source.stream()
+                .map(entry -> convert(entry, ManualRedactionEntry.class, new ManualRedactionEntryMapper()))
+                .collect(Collectors.toSet());
     }
 
 
     @Transactional
-    public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId, List<RemoveRedactionRequest> removeRedactionRequests) {
+    public List<ManualAddResponse> addRemoveRedaction(String dossierId,
+                                                      String fileId,
+                                                      Set<RemoveRedactionRequestModel> removeRedactionRequests,
+                                                      String dossierTemplateId,
+                                                      boolean includeUnprocessed) {
 
         var response = new ArrayList<ManualAddResponse>();
+        List<RemoveRedactionRequest> requests = manualRedactionMapper.toRemoveRedactionRequestList(dossierId,
+                                                                                                   fileId,
+                                                                                                   dossierTemplateId,
+                                                                                                   removeRedactionRequests,
+                                                                                                   includeUnprocessed);
 
         //validate removing from dossier template dictionary
-        for (RemoveRedactionRequest removeRedactionRequest : removeRedactionRequests) {
+        for (RemoveRedactionRequest removeRedactionRequest : requests) {
             manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest,
-                    removeRedactionRequest.getTypeToRemove(),
+                                                                                 removeRedactionRequest.getTypeToRemove(),
-                    removeRedactionRequest.getDossierTemplateId());
+                                                                                 removeRedactionRequest.getDossierTemplateId());
 
             log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
             removeRedactionPersistenceService.insert(fileId, removeRedactionRequest);
 
             Long commentId = commentService.addCommentAndGetId(fileId,
-                    removeRedactionRequest.getAnnotationId(),
+                                                               removeRedactionRequest.getAnnotationId(),
-                    removeRedactionRequest.getComment(),
+                                                               removeRedactionRequest.getComment(),
-                    removeRedactionRequest.getUser());
+                                                               removeRedactionRequest.getUser());
 
             Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(fileId, removeRedactionRequest);
 
             boolean removedFromDictionary = !typeIdsOfModifiedDictionaries.isEmpty();
 
-            removeRedactionPersistenceService.updateModifiedDictionaries(fileId,
+            removeRedactionPersistenceService.updateModifiedDictionaries(fileId, removeRedactionRequest.getAnnotationId(), removedFromDictionary, typeIdsOfModifiedDictionaries);
-                    removeRedactionRequest.getAnnotationId(),
-                    removedFromDictionary,
-                    typeIdsOfModifiedDictionaries);
 
             response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
         }
@@ -167,18 +196,19 @@ public class ManualRedactionService {
 
 
     @Transactional
-    public List<ManualAddResponse> addForceRedaction(String dossierId, String fileId, List<ForceRedactionRequest> forceRedactionRequests) {
+    public List<ManualAddResponse> addForceRedaction(String dossierId, String fileId, Set<ForceRedactionRequestModel> forceRedactionRequests) {
 
         var response = new ArrayList<ManualAddResponse>();
         dossierPersistenceService.getAndValidateDossier(dossierId);
+        List<ForceRedactionRequest> requests = manualRedactionMapper.toForceRedactionRequestList(dossierId, fileId, forceRedactionRequests, getEntityLogEntryConsumer(fileId));
 
-        for (var forceRedactionRequest : forceRedactionRequests) {
+        for (var forceRedactionRequest : requests) {
             forceRedactionPersistenceService.insert(fileId, forceRedactionRequest);
 
             Long commentId = commentService.addCommentAndGetId(fileId,
-                    forceRedactionRequest.getAnnotationId(),
+                                                               forceRedactionRequest.getAnnotationId(),
-                    forceRedactionRequest.getComment(),
+                                                               forceRedactionRequest.getComment(),
-                    forceRedactionRequest.getUser());
+                                                               forceRedactionRequest.getUser());
 
             response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
         }
@@ -191,19 +221,24 @@ public class ManualRedactionService {
     }
 
 
+    @Deprecated(forRemoval = true)
     @Transactional
-    public List<ManualAddResponse> addLegalBasisChange(String dossierId, String fileId, List<LegalBasisChangeRequest> legalBasisChangeRequests) {
+    public List<ManualAddResponse> addLegalBasisChange(String dossierId, String fileId, Set<LegalBasisChangeRequestModel> legalBasisChangeRequests) {
 
         var response = new ArrayList<ManualAddResponse>();
         dossierPersistenceService.getAndValidateDossier(dossierId);
+        List<LegalBasisChangeRequest> requests = manualRedactionMapper.toLegalBasisChangeRequestList(dossierId,
+                                                                                                     fileId,
+                                                                                                     legalBasisChangeRequests,
+                                                                                                     getEntityLogEntryConsumer(fileId));
 
-        for (var legalBasisChangeRequest : legalBasisChangeRequests) {
+        for (var legalBasisChangeRequest : requests) {
             legalBasisChangePersistenceService.insert(fileId, legalBasisChangeRequest);
 
             Long commentId = commentService.addCommentAndGetId(fileId,
-                    legalBasisChangeRequest.getAnnotationId(),
+                                                               legalBasisChangeRequest.getAnnotationId(),
-                    legalBasisChangeRequest.getComment(),
+                                                               legalBasisChangeRequest.getComment(),
-                    legalBasisChangeRequest.getUser());
+                                                               legalBasisChangeRequest.getUser());
 
             response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build());
         }
@@ -215,35 +250,51 @@ public class ManualRedactionService {
 
 
     @Transactional
-    public List<ManualAddResponse> addRecategorization(String dossierId, String fileId, List<RecategorizationRequest> recategorizationRequests) {
+    public List<ManualAddResponse> addRecategorization(String dossierId,
+                                                       String fileId,
+                                                       Set<RecategorizationRequestModel> recategorizationRequests,
+                                                       String dossierTemplateId,
+                                                       boolean includeUnprocessed) {
 
         var response = new ArrayList<ManualAddResponse>();
-        for (var recategorizationRequest : recategorizationRequests) {
+        dossierPersistenceService.getAndValidateDossier(dossierId);
+        List<RecategorizationRequest> requests = manualRedactionMapper.toRecategorizationRequestList(dossierId,
+                                                                                                     fileId,
+                                                                                                     dossierTemplateId,
+                                                                                                     recategorizationRequests,
+                                                                                                     includeUnprocessed,
+                                                                                                     getEntityLogEntryConsumer(fileId));
+
+        for (var recategorizationRequest : requests) {
             manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(recategorizationRequest, recategorizationRequest.getValue());
             manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(recategorizationRequest,
-                    recategorizationRequest.getTypeToRemove(),
+                                                                                 recategorizationRequest.getTypeToRemove(),
-                    recategorizationRequest.getDossierTemplateId());
+                                                                                 recategorizationRequest.getDossierTemplateId());
+
+            checkLegalBasisLength(recategorizationRequest.getLegalBasis());
 
             recategorizationPersistenceService.insert(fileId, recategorizationRequest);
 
             Set<String> typeIdsOfDictionariesWithAdd = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
-                    recategorizationRequest.getValue(),
+                                                                                                                                            recategorizationRequest.getValue(),
-                    recategorizationRequest);
+                                                                                                                                            recategorizationRequest);
+
             Set<String> typeIdsOfDictionariesWithDelete = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(fileId,
-                    recategorizationRequest);
+                                                                                                                                                    recategorizationRequest);
 
             recategorizationPersistenceService.updateModifiedDictionaries(fileId,
-                    recategorizationRequest.getAnnotationId(),
+                                                                          recategorizationRequest.getAnnotationId(),
-                    typeIdsOfDictionariesWithAdd,
+                                                                          typeIdsOfDictionariesWithAdd,
-                    typeIdsOfDictionariesWithDelete);
+                                                                          typeIdsOfDictionariesWithDelete);
 
             Long commentId = commentService.addCommentAndGetId(fileId,
-                    recategorizationRequest.getAnnotationId(),
+                                                               recategorizationRequest.getAnnotationId(),
-                    recategorizationRequest.getComment(),
+                                                               recategorizationRequest.getComment(),
-                    recategorizationRequest.getUser());
+                                                               recategorizationRequest.getUser());
 
             response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).build());
         }
+
         reprocess(dossierId, fileId);
 
         fileStatusPersistenceService.setLastManualChangeDate(fileId, OffsetDateTime.now());
@@ -252,33 +303,47 @@ public class ManualRedactionService {
     }
 
 
+    private void checkLegalBasisLength(String legalBasis) {
+
+        if (legalBasis.length() > MAX_LEGAL_BASIS_LENGTH) {
+            throw new BadRequestException(String.format("The legal basis is too long (%s), max length %s", legalBasis.length(), MAX_LEGAL_BASIS_LENGTH));
+        }
+    }
+
+
     @Transactional
     @Observed(name = "ManualRedactionService", contextualName = "add-manual-redaction")
-    public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests, boolean includeUnprocessed) {
+    public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, Set<ResizeRedactionRequestModel> resizeRedactionRequests, boolean includeUnprocessed) {
 
         List<ManualAddResponse> response = new ArrayList<>();
         List<ManualResizeRedactionEntity> manualResizeRedactionEntities = new ArrayList<>();
         EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId, Collections.emptyList(), includeUnprocessed);
+        List<ResizeRedactionRequest> requests = manualRedactionMapper.toResizeRedactionRequestList(resizeRedactionRequests, entityLog,
+                                                                                                   getEntityLogEntryConsumer(fileId));
 
-        for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) {
+        for (ResizeRedactionRequest resizeRedactionRequest : requests) {
 
             var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
             manualResizeRedactionEntities.add(resizeRedaction);
 
             if (resizeRedactionRequest.getComment() != null) {
                 Long commentId = commentService.addCommentAndGetId(fileId,
-                        resizeRedactionRequest.getAnnotationId(),
+                                                                   resizeRedactionRequest.getAnnotationId(),
-                        resizeRedactionRequest.getComment(),
+                                                                   resizeRedactionRequest.getComment(),
-                        resizeRedactionRequest.getUser());
+                                                                   resizeRedactionRequest.getUser());
                 response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build());
             }
 
             Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.updateDictionaryForResizeRedactions(dossierId,
-                    fileId,
+                                                                                                                                   fileId,
-                    resizeRedaction,
+                                                                                                                                   resizeRedaction,
-                    getEntityLogEntry(entityLog, resizeRedaction.getId().getAnnotationId()));
+                                                                                                                                   getEntityLogEntry(entityLog,
+                                                                                                                                                     resizeRedaction.getId()
+                                                                                                                                                             .getAnnotationId()));
 
-            resizeRedactionPersistenceService.updateModifiedDictionaries(resizeRedaction.getId().getFileId(), resizeRedaction.getId().getAnnotationId(), typeIdsOfModifiedDictionaries);
+            resizeRedactionPersistenceService.updateModifiedDictionaries(resizeRedaction.getId().getFileId(),
+                                                                         resizeRedaction.getId().getAnnotationId(),
+                                                                         typeIdsOfModifiedDictionaries);
         }
 
         manualResizeRedactionEntities = manualResizeRedactionEntities.stream()
@@ -301,14 +366,20 @@ public class ManualRedactionService {
 
     private Set<ManualResizeRedaction> convertResizeRedactions(List<ManualResizeRedactionEntity> source) {
 
-        return source.stream().map(entry -> convert(entry, ManualResizeRedaction.class, new ManualResizeRedactionMapper())).collect(Collectors.toSet());
+        return source.stream()
+                .map(entry -> convert(entry, ManualResizeRedaction.class, new ManualResizeRedactionMapper()))
+                .collect(Collectors.toSet());
     }
 
 
     private void validatePositions(String fileId, AddRedactionRequest addRedactionRequest) {
 
         var numberOfPages = fileStatusService.getStatus(fileId).getNumberOfPages();
-        addRedactionRequest.getPositions().stream().filter(p -> p.getPage() > numberOfPages).findAny().ifPresent(p -> new BadRequestException("Invalid page found in the request"));
+        addRedactionRequest.getPositions()
+                .stream()
+                .filter(p -> p.getPage() > numberOfPages)
+                .findAny()
+                .ifPresent(p -> new BadRequestException("Invalid page found in the request"));
     }
 
 
@@ -328,9 +399,9 @@ public class ManualRedactionService {
     }
 
 
-    public ManualRedactions getManualRedactions(String fileId, boolean unprocessed) {
+    public ManualRedactions getManualRedactions(String fileId, ManualChangesQueryOptions options) {
 
-        return manualRedactionProviderService.getManualRedactions(fileId, unprocessed);
+        return manualRedactionProviderService.getManualRedactions(fileId, options);
     }
 
 
@@ -339,49 +410,134 @@ public class ManualRedactionService {
 
         if (manualRedactions != null) {
 
-            if (manualRedactions.getEntriesToAdd() != null) {
+            manualRedactions.getEntriesToAdd()
-                manualRedactions.getEntriesToAdd().forEach(e -> {
+                    .forEach(e -> {
-                    if (e.getProcessedDate() == null) {
+                        if (e.getProcessedDate() == null) {
-                        addRedactionPersistenceService.markAsProcessed(e);
+                            addRedactionPersistenceService.markAsProcessed(e);
-                    }
+                        }
-                });
+                    });
-            }
+
-            if (manualRedactions.getIdsToRemove() != null) {
+            manualRedactions.getIdsToRemove()
-                manualRedactions.getIdsToRemove().forEach(e -> {
+                    .forEach(e -> {
-                    if (e.getProcessedDate() == null) {
+                        if (e.getProcessedDate() == null) {
-                        removeRedactionPersistenceService.markAsProcessed(e);
+                            removeRedactionPersistenceService.markAsProcessed(e);
-                    }
+                        }
-                });
+                    });
-            }
+
-            if (manualRedactions.getForceRedactions() != null) {
+            manualRedactions.getForceRedactions()
-                manualRedactions.getForceRedactions().forEach(e -> {
+                    .forEach(e -> {
-                    if (e.getProcessedDate() == null) {
+                        if (e.getProcessedDate() == null) {
-                        forceRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
+                            forceRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
-                    }
+                        }
-                });
+                    });
-            }
+
-            if (manualRedactions.getRecategorizations() != null) {
+            manualRedactions.getRecategorizations()
-                manualRedactions.getRecategorizations().forEach(e -> {
+                    .forEach(e -> {
-                    if (e.getProcessedDate() == null) {
+                        if (e.getProcessedDate() == null) {
-                        recategorizationPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
+                            recategorizationPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
-                    }
+                        }
-                });
+                    });
-            }
+
-            if (manualRedactions.getResizeRedactions() != null) {
+            manualRedactions.getResizeRedactions()
-                manualRedactions.getResizeRedactions().forEach(e -> {
+                    .forEach(e -> {
-                    if (e.getProcessedDate() == null) {
+                        if (e.getProcessedDate() == null) {
-                        resizeRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
+                            resizeRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
-                    }
+                        }
-                });
+                    });
-            }
+
-            if (manualRedactions.getLegalBasisChanges() != null) {
+            manualRedactions.getLegalBasisChanges()
-                manualRedactions.getLegalBasisChanges().forEach(e -> {
+                    .forEach(e -> {
-                    if (e.getProcessedDate() == null) {
+                        if (e.getProcessedDate() == null) {
-                        legalBasisChangePersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
+                            legalBasisChangePersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
-                    }
+                        }
-                });
+                    });
-            }
+
         }
     }
 
+
+    public int addManualRedactionEntries(List<ManualRedactionEntry> manualRedactionEntriesToAdd, boolean addAsProcessed) {
+
+        List<ManualRedactionEntryEntity> manualRedactionEntryEntities = manualRedactionEntriesToAdd.stream()
+                .map(this::toManualRedactionEntryEntity)
+                .toList();
+
+        if (addAsProcessed) {
+            manualRedactionEntryEntities.forEach(e -> e.setProcessedDate(OffsetDateTime.now()));
+        }
+
+        return addRedactionPersistenceService.insert(manualRedactionEntryEntities).size();
+    }
+
+
+    private ManualRedactionEntryEntity toManualRedactionEntryEntity(ManualRedactionEntry manualRedactionEntry) {
+
+        var file = fileStatusPersistenceService.getStatus(manualRedactionEntry.getFileId());
+        String dossierId = file.getDossierId();
+        String dossierTemplateId = dossierPersistenceService.getDossierTemplateId(dossierId);
+        String type = manualRedactionEntry.getType();
+        ManualRedactionEntryEntity manualRedactionEntryEntity = MagicConverter.convert(manualRedactionEntry, ManualRedactionEntryEntity.class);
+        String typeId;
+        if (manualRedactionEntry.isAddToDossierDictionary()) {
+            typeId = TypeIdUtils.toTypeId(type, dossierTemplateId, dossierId);
+        } else {
+            typeId = TypeIdUtils.toTypeId(type, dossierTemplateId);
+        }
+        manualRedactionEntryEntity.setTypeId(typeId);
+        manualRedactionEntryEntity.setTypeIdsOfModifiedDictionaries(Set.of(typeId));
+        manualRedactionEntryEntity.setId(new AnnotationEntityId(manualRedactionEntry.getAnnotationId(), manualRedactionEntry.getFileId()));
+        manualRedactionEntryEntity.setPositions(manualRedactionEntry.getPositions()
+                                                        .stream()
+                                                        .map(p -> MagicConverter.convert(p, RectangleEntity.class))
+                                                        .toList());
+        manualRedactionEntryEntity.setRequestDate(manualRedactionEntry.getRequestDate());
+        manualRedactionEntryEntity.setFileStatus(file);
+        return manualRedactionEntryEntity;
+    }
+
+
+    public void deleteManualRedactionEntries(List<ManualRedactionEntry> list) {
+
+        list.forEach(manualRedactionEntry -> addRedactionPersistenceService.hardDelete(manualRedactionEntry.getFileId(), manualRedactionEntry.getAnnotationId()));
+    }
+
+
+    private void addManualRedactionEntry(String fileId, EntityLogEntry entityLogEntry) {
+
+        ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
+                .value(entityLogEntry.getValue())
+                .reason(entityLogEntry.getReason())
+                .section(entityLogEntry.getSection())
+                .annotationId(entityLogEntry.getId())
+                .type(entityLogEntry.getType())
+                .addToDossierDictionary(false)
+                .addToDictionary(false)
+                .positions(convertPositions(entityLogEntry.getPositions()))
+                .rectangle(entityLogEntry.getEntryType() == EntryType.AREA)
+                .user(KeycloakSecurity.getUserId())
+                .legalBasis(entityLogEntry.getLegalBasis())
+                .textAfter(entityLogEntry.getTextAfter())
+                .textBefore(entityLogEntry.getTextBefore())
+                .dictionaryEntryType(getDictionaryEntryType(entityLogEntry))
+                .fileId(fileId)
+                .requestDate(OffsetDateTime.now())
+                .build();
+
+        addManualRedactionEntries(List.of(manualRedactionEntry), false);
+    }
+
+
+    private List<Rectangle> convertPositions(List<Position> positions) {
+
+        return positions.stream()
+                .map(rectangle -> new Rectangle(rectangle.x(), rectangle.y(), rectangle.w(), rectangle.h(), rectangle.getPageNumber()))
+                .toList();
+    }
+
+
+    private Consumer<EntityLogEntry> getEntityLogEntryConsumer(String fileId) {
+
+        return entry -> addManualRedactionEntry(fileId, entry);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionUndoService.java

@@ -21,6 +21,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRecategorizationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
@@ -96,7 +97,7 @@ public class ManualRedactionUndoService {
 
     private ManualRedactions getManualRedactions(String fileId) {
 
-        return manualRedactionProviderService.getManualRedactions(fileId);
+        return manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted());
     }
 
 
@@ -112,16 +113,21 @@ public class ManualRedactionUndoService {
                 .stream()
                 .filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualResizeRedaction)
                 .map(ManualRedactionWrapperModel::getId)
-                .collect(Collectors.toList());
+                .toList();
         if (!manualResizeRedactions.isEmpty()) {
             deleteResizeRedaction(dossierId, fileId, manualResizeRedactions);
             manualResizeRedactions.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
-                    .userId(KeycloakSecurity.getUserId())
+                                                                                                 .userId(KeycloakSecurity.getUserId())
-                    .objectId(fileId)
+                                                                                                 .objectId(fileId)
-                    .category(AuditCategory.DOCUMENT.name())
+                                                                                                 .category(AuditCategory.DOCUMENT.name())
-                    .message("Undo of manual resize redaction was done.")
+                                                                                                 .message("Undo of manual resize redaction was done.")
-                    .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
+                                                                                                 .details(Map.of(DOSSIER_ID,
-                    .build()));
+                                                                                                                 dossierId,
+                                                                                                                 FILE_ID,
+                                                                                                                 fileId,
+                                                                                                                 ANNOTATION_ID,
+                                                                                                                 annotationId))
+                                                                                                 .build()));
         }
     }
 
@@ -142,17 +148,22 @@ public class ManualRedactionUndoService {
                 .stream()
                 .filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualLegalBasisChange)
                 .map(ManualRedactionWrapperModel::getId)
-                .collect(Collectors.toList());
+                .toList();
         if (!manualLegalBasisChanges.isEmpty()) {
 
             deleteLegalBasisChange(dossierId, fileId, manualLegalBasisChanges);
             manualLegalBasisChanges.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
-                    .userId(KeycloakSecurity.getUserId())
+                                                                                                  .userId(KeycloakSecurity.getUserId())
-                    .objectId(fileId)
+                                                                                                  .objectId(fileId)
-                    .category(AuditCategory.DOCUMENT.name())
+                                                                                                  .category(AuditCategory.DOCUMENT.name())
-                    .message("Undo of legal basis change was done.")
+                                                                                                  .message("Undo of legal basis change was done.")
-                    .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
+                                                                                                  .details(Map.of(DOSSIER_ID,
-                    .build()));
+                                                                                                                  dossierId,
+                                                                                                                  FILE_ID,
+                                                                                                                  fileId,
+                                                                                                                  ANNOTATION_ID,
+                                                                                                                  annotationId))
+                                                                                                  .build()));
         }
     }
 
@@ -168,21 +179,26 @@ public class ManualRedactionUndoService {
 
     private void undoRecategorization(String dossierId, String fileId, Map<String, ManualRedactionWrapperModel> manualRedactionWrappers, boolean includeUnprocessed) {
 
-        List<String> manualImageRecategorizations = manualRedactionWrappers.values()
+        List<String> manualRecategorizations = manualRedactionWrappers.values()
                 .stream()
                 .filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualRecategorization)
                 .map(ManualRedactionWrapperModel::getId)
-                .collect(Collectors.toList());
+                .toList();
-        if (!manualImageRecategorizations.isEmpty()) {
+        if (!manualRecategorizations.isEmpty()) {
 
-            deleteRecategorization(dossierId, fileId, manualImageRecategorizations, includeUnprocessed);
+            deleteRecategorization(dossierId, fileId, manualRecategorizations, includeUnprocessed);
-            manualImageRecategorizations.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
+            manualRecategorizations.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
-                    .userId(KeycloakSecurity.getUserId())
+                                                                                                       .userId(KeycloakSecurity.getUserId())
-                    .objectId(fileId)
+                                                                                                       .objectId(fileId)
-                    .category(AuditCategory.DOCUMENT.name())
+                                                                                                       .category(AuditCategory.DOCUMENT.name())
-                    .message("Undo of manual image recategorization was done.")
+                                                                                                       .message("Undo of manual recategorization was done.")
-                    .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
+                                                                                                       .details(Map.of(DOSSIER_ID,
-                    .build()));
+                                                                                                                       dossierId,
+                                                                                                                       FILE_ID,
+                                                                                                                       fileId,
+                                                                                                                       ANNOTATION_ID,
+                                                                                                                       annotationId))
+                                                                                                       .build()));
         }
     }
 
@@ -197,10 +213,10 @@ public class ManualRedactionUndoService {
             String originalValue = getEntityLogEntry(entityLog, annotationId).getValue();
             manualRedactionDictionaryUpdateHandler.revertRemoveFromDictionary(originalValue, dossierId, fileId, recategorizationEntity.getTypeIdsOfDictionariesWithDelete());
             manualRedactionDictionaryUpdateHandler.revertAddToDictionary(originalValue,
-                    DictionaryEntryType.ENTRY,
+                                                                         DictionaryEntryType.ENTRY,
-                    fileId,
+                                                                         fileId,
-                    dossierId,
+                                                                         dossierId,
-                    recategorizationEntity.getTypeIdsOfDictionariesWithAdd());
+                                                                         recategorizationEntity.getTypeIdsOfDictionariesWithAdd());
             recategorizationPersistenceService.updateModifiedDictionaries(fileId, annotationId, Collections.emptySet(), Collections.emptySet());
             recategorizationPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
         }
@@ -214,17 +230,22 @@ public class ManualRedactionUndoService {
                 .stream()
                 .filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualForceRedaction)
                 .map(ManualRedactionWrapperModel::getId)
-                .collect(Collectors.toList());
+                .toList();
         if (!manualForceRedactions.isEmpty()) {
 
             deleteForceRedaction(dossierId, fileId, manualForceRedactions);
             manualForceRedactions.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
-                    .userId(KeycloakSecurity.getUserId())
+                                                                                                .userId(KeycloakSecurity.getUserId())
-                    .objectId(fileId)
+                                                                                                .objectId(fileId)
-                    .category(AuditCategory.DOCUMENT.name())
+                                                                                                .category(AuditCategory.DOCUMENT.name())
-                    .message("Undo of manual force redaction was done.")
+                                                                                                .message("Undo of manual force redaction was done.")
-                    .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
+                                                                                                .details(Map.of(DOSSIER_ID,
-                    .build()));
+                                                                                                                dossierId,
+                                                                                                                FILE_ID,
+                                                                                                                fileId,
+                                                                                                                ANNOTATION_ID,
+                                                                                                                annotationId))
+                                                                                                .build()));
         }
     }
 
@@ -246,16 +267,16 @@ public class ManualRedactionUndoService {
                 .stream()
                 .filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof IdRemoval)
                 .map(ManualRedactionWrapperModel::getId)
-                .collect(Collectors.toList());
+                .toList();
         if (!idRemovals.isEmpty()) {
             deleteRemoveRedaction(dossierId, fileId, idRemovals);
             idRemovals.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
-                    .userId(KeycloakSecurity.getUserId())
+                                                                                     .userId(KeycloakSecurity.getUserId())
-                    .objectId(fileId)
+                                                                                     .objectId(fileId)
-                    .category(AuditCategory.DOCUMENT.name())
+                                                                                     .category(AuditCategory.DOCUMENT.name())
-                    .message("Undo of manual remove redaction was done.")
+                                                                                     .message("Undo of manual remove redaction was done.")
-                    .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
+                                                                                     .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
-                    .build()));
+                                                                                     .build()));
         }
     }
 
@@ -283,16 +304,21 @@ public class ManualRedactionUndoService {
                 .stream()
                 .filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualRedactionEntry)
                 .map(ManualRedactionWrapperModel::getId)
-                .collect(Collectors.toList());
+                .toList();
         if (!manualRedactionEntries.isEmpty()) {
             deleteAddRedaction(dossierId, fileId, manualRedactionEntries);
             manualRedactionEntries.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
-                    .userId(KeycloakSecurity.getUserId())
+                                                                                                 .userId(KeycloakSecurity.getUserId())
-                    .objectId(fileId)
+                                                                                                 .objectId(fileId)
-                    .category(AuditCategory.DOCUMENT.name())
+                                                                                                 .category(AuditCategory.DOCUMENT.name())
-                    .message("Undo of manual add redaction was done.")
+                                                                                                 .message("Undo of manual add redaction was done.")
-                    .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
+                                                                                                 .details(Map.of(DOSSIER_ID,
-                    .build()));
+                                                                                                                 dossierId,
+                                                                                                                 FILE_ID,
+                                                                                                                 fileId,
+                                                                                                                 ANNOTATION_ID,
+                                                                                                                 annotationId))
+                                                                                                 .build()));
         }
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/PendingDictionaryEntryFactory.java

@@ -0,0 +1,249 @@
+package com.iqser.red.service.persistence.management.v1.processor.service.manualredactions;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
+
+@Service
+public class PendingDictionaryEntryFactory {
+
+    public EntityLogEntry buildAddToDictionaryEntry(ManualRedactionEntry manualRedactionEntry) {
+
+        var manualChanges = List.of(ManualChange.builder()
+                                            .manualRedactionType(ManualRedactionType.ADD_TO_DICTIONARY)
+                                            .requestedDate(manualRedactionEntry.getRequestDate())
+                                            .processedDate(null)
+                                            .userId(manualRedactionEntry.getUser())
+                                            .propertyChanges(Map.of("value", manualRedactionEntry.getValue()))
+                                            .build());
+        return EntityLogEntry.builder()
+                .id(manualRedactionEntry.getAnnotationId())
+                .value(manualRedactionEntry.getValue())
+                .type(manualRedactionEntry.getType())
+                .entryType(Optional.ofNullable(manualRedactionEntry.getDictionaryEntryType())
+                                   .orElse(DictionaryEntryType.ENTRY).toEntryType())
+                .state(EntryState.PENDING)
+                .dictionaryEntry(manualRedactionEntry.isAddToDictionary())
+                .dossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
+                .reason("Pending add to dictionary.")
+                .legalBasis("Pending add to dictionary.")
+                .matchedRule("")
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .section("")
+                .positions(convertPositions(manualRedactionEntry.getPositions()))
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .changes(Collections.emptyList())
+                .manualChanges(manualChanges)
+                .engines(Set.of(Engine.DICTIONARY))
+                .reference(Collections.emptySet())
+                .importedRedactionIntersections(Collections.emptySet())
+                .build();
+    }
+
+
+    private List<Position> convertPositions(List<Rectangle> rectangles) {
+
+        return rectangles.stream()
+                .map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage()))
+                .collect(Collectors.toList());
+    }
+
+
+    public EntityLogEntry buildRemoveFromDictionary(IdRemoval manualChange, EntityLogEntry originalEntry) {
+
+        var manualChanges = List.of(ManualChange.builder()
+                                            .manualRedactionType(ManualRedactionType.REMOVE_FROM_DICTIONARY)
+                                            .requestedDate(manualChange.getRequestDate())
+                                            .processedDate(manualChange.getProcessedDate())
+                                            .userId(manualChange.getUser())
+                                            .propertyChanges(Map.of("remove", originalEntry.getValue()))
+                                            .build());
+
+        return EntityLogEntry.builder()
+                .id(originalEntry.getId())
+                .value(originalEntry.getValue())
+                .type(originalEntry.getType())
+                .entryType(originalEntry.getEntryType())
+                .state(EntryState.PENDING)
+                .dictionaryEntry(manualChange.isRemoveFromDictionary())
+                .dossierDictionaryEntry(!manualChange.isRemoveFromAllDossiers())
+                .reason("Pending remove from dictionary.")
+                .legalBasis("Pending remove from dictionary.")
+                .matchedRule("")
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .section("")
+                .positions(originalEntry.getPositions())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .changes(Collections.emptyList())
+                .manualChanges(manualChanges)
+                .engines(Set.of(Engine.DICTIONARY))
+                .reference(Collections.emptySet())
+                .importedRedactionIntersections(Collections.emptySet())
+                .build();
+    }
+
+
+    public EntityLogEntry buildResizeWithDictionary(ManualResizeRedaction manualChange, EntityLogEntry originalEntry) {
+
+        Map<String, String> propertyChanges = buildPropertyChangesForManualResize(manualChange, originalEntry);
+
+        var manualChanges = List.of(ManualChange.builder()
+                                            .manualRedactionType(ManualRedactionType.RESIZE_IN_DICTIONARY)
+                                            .requestedDate(manualChange.getRequestDate())
+                                            .processedDate(manualChange.getProcessedDate())
+                                            .userId(manualChange.getUser())
+                                            .propertyChanges(propertyChanges)
+                                            .build());
+        return EntityLogEntry.builder()
+                .id(originalEntry.getId())
+                .value(manualChange.getValue())
+                .type(originalEntry.getType())
+                .entryType(originalEntry.getEntryType())
+                .state(EntryState.PENDING)
+                .dictionaryEntry(manualChange.getUpdateDictionary())
+                .dossierDictionaryEntry(!manualChange.isAddToAllDossiers())
+                .reason("Pending resize with dictionary.")
+                .legalBasis("Pending resize with dictionary.")
+                .matchedRule("")
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .section("")
+                .positions(convertPositions(manualChange.getPositions()))
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .changes(Collections.emptyList())
+                .manualChanges(manualChanges)
+                .engines(Set.of(Engine.DICTIONARY))
+                .reference(Collections.emptySet())
+                .importedRedactionIntersections(Collections.emptySet())
+                .build();
+    }
+
+
+    private static Map<String, String> buildPropertyChangesForManualResize(ManualResizeRedaction manualChange, EntityLogEntry originalEntry) {
+
+        Map<String, String> prop;
+        if (manualChange.getValue().length() >= originalEntry.getValue().length()) {
+            prop = Map.of("add", manualChange.getValue());
+        } else {
+            prop = Map.of("add", manualChange.getValue(), "remove", originalEntry.getValue());
+        }
+        return prop;
+    }
+
+
+    public EntityLogEntry buildRecategorizeWithDictionary(ManualRecategorization manualChange, EntityLogEntry originalEntry) {
+
+        var manualChanges = List.of(ManualChange.builder()
+                                            .manualRedactionType(ManualRedactionType.RECATEGORIZE_IN_DICTIONARY)
+                                            .requestedDate(manualChange.getRequestDate())
+                                            .processedDate(manualChange.getProcessedDate())
+                                            .userId(manualChange.getUser())
+                                            .propertyChanges(Map.of("type",
+                                                                    manualChange.getType(),
+                                                                    "legalBasis",
+                                                                    manualChange.getLegalBasis(),
+                                                                    "section",
+                                                                    manualChange.getSection(),
+                                                                    "value",
+                                                                    manualChange.getValue()))
+                                            .build());
+
+        return EntityLogEntry.builder()
+                .id(originalEntry.getId())
+                .value(originalEntry.getValue())
+                .type(manualChange.getType())
+                .entryType(originalEntry.getEntryType())
+                .state(EntryState.PENDING)
+                .dictionaryEntry(manualChange.isAddToDictionary())
+                .dossierDictionaryEntry(!manualChange.isAddToAllDossiers())
+                .reason("Pending recategorize with dictionary.")
+                .legalBasis("Pending recategorize with dictionary.")
+                .matchedRule("")
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .section("")
+                .positions(originalEntry.getPositions())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .changes(Collections.emptyList())
+                .manualChanges(manualChanges)
+                .engines(Set.of(Engine.DICTIONARY))
+                .reference(Collections.emptySet())
+                .importedRedactionIntersections(Collections.emptySet())
+                .build();
+    }
+
+
+
+    public EntityLogEntry buildPendingImageRecategorizationEntry(ManualRecategorization manualChange, EntityLogEntry originalEntry) {
+
+        var manualChanges = List.of(ManualChange.builder()
+                                            .manualRedactionType(ManualRedactionType.RECATEGORIZE)
+                                            .requestedDate(manualChange.getRequestDate())
+                                            .processedDate(manualChange.getProcessedDate())
+                                            .userId(manualChange.getUser())
+                                            .propertyChanges(EntityLogMergeService.getPropertyChanges(manualChange))
+                                            .build());
+
+        String reason = String.format("Image has been recategorized from %s to %s", originalEntry.getType(), manualChange.getType());
+
+        return EntityLogEntry.builder()
+                .id(originalEntry.getId())
+                .value(originalEntry.getValue())
+                .type(manualChange.getType())
+                .entryType(originalEntry.getEntryType())
+                .state(EntryState.PENDING)
+                .dictionaryEntry(manualChange.isAddToDictionary())
+                .dossierDictionaryEntry(manualChange.isAddToAllDossiers())
+                .reason(reason)
+                .legalBasis(reason)
+                .matchedRule("")
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .section("")
+                .positions(originalEntry.getPositions())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .changes(Collections.emptyList())
+                .manualChanges(manualChanges)
+                .engines(Set.of(Engine.MANUAL))
+                .reference(Collections.emptySet())
+                .importedRedactionIntersections(Collections.emptySet())
+                .build();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java

@@ -11,13 +11,14 @@ import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionarySummaryResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.TypeRankSummary;
 
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
@@ -85,18 +86,20 @@ public class DictionaryPersistenceService {
 
     private void checkRankAlreadyExists(String type, String dossierTemplateId, int rank, String dossierId) {
 
-        Optional<TypeEntity> existingTypeValueForRank = getTypeForRank(dossierTemplateId, rank, dossierId);
+        List<TypeEntity> existingTypesForRank = getTypesForRank(dossierTemplateId, rank, dossierId);
-
+        if (!existingTypesForRank.isEmpty()) {
-        if (existingTypeValueForRank.isPresent() && !existingTypeValueForRank.get().isDeleted() && !existingTypeValueForRank.get().getType().equalsIgnoreCase(type)) {
+            for (var existingTypeValueForRank : existingTypesForRank) {
-            throw new BadRequestException("Rank already exists: " + rank + " on type: " + existingTypeValueForRank.get().getType());
+                if (!existingTypeValueForRank.isDeleted() && !existingTypeValueForRank.getType().equalsIgnoreCase(type)) {
+                    throw new ConflictException("Rank already exists: " + rank + " on type: " + existingTypeValueForRank.getType() + " type id: " + existingTypeValueForRank.getId());
+                }
+            }
         }
-
     }
 
 
-    private Optional<TypeEntity> getTypeForRank(String dossierTemplateId, int rank, String dossierId) {
+    private List<TypeEntity> getTypesForRank(String dossierTemplateId, int rank, String dossierId) {
 
-        return typeRepository.findOneByDossierTemplateIdAndDossierIdAndRank(dossierTemplateId, dossierId, rank);
+        return typeRepository.findByDossierTemplateIdAndDossierIdAndRank(dossierTemplateId, dossierId, rank);
 
     }
 
@@ -104,37 +107,41 @@ public class DictionaryPersistenceService {
     @Transactional
     public void updateType(String typeId, TypeEntity typeValueRequest) {
 
-        typeRepository.findById(typeId).ifPresent(type -> {
+        typeRepository.findById(typeId)
+                .ifPresent(type -> {
 //            if (type.isDeleted()) {
 //                throw new NotFoundException("Type is deleted!");
 //            }
 
-            type.setVersion(type.getVersion() + 1);
+                    type.setVersion(type.getVersion() + 1);
-            checkRankAlreadyExists(type.getType(), type.getDossierTemplate().getId(), typeValueRequest.getRank(), type.getDossier() == null ? null : type.getDossier().getId());
+                    checkRankAlreadyExists(type.getType(),
-            if (type.isSystemManaged()) {
+                                           type.getDossierTemplate().getId(),
-                type.setHexColor(typeValueRequest.getHexColor());
+                                           typeValueRequest.getRank(),
-                type.setRecommendationHexColor(typeValueRequest.getRecommendationHexColor());
+                                           type.getDossier() == null ? null : type.getDossier().getId());
-                type.setSkippedHexColor(typeValueRequest.getSkippedHexColor());
+                    if (type.isSystemManaged()) {
-                type.setDescription(typeValueRequest.getDescription());
+                        type.setHexColor(typeValueRequest.getHexColor());
-                type.setLabel(typeValueRequest.getLabel());
+                        type.setRecommendationHexColor(typeValueRequest.getRecommendationHexColor());
-                type.setAddToDictionaryAction(typeValueRequest.isAddToDictionaryAction());
+                        type.setSkippedHexColor(typeValueRequest.getSkippedHexColor());
-            } else {
+                        type.setDescription(typeValueRequest.getDescription());
-                BeanUtils.copyProperties(typeValueRequest,
+                        type.setLabel(typeValueRequest.getLabel());
-                        type,
+                        type.setAddToDictionaryAction(typeValueRequest.isAddToDictionaryAction());
-                        "type",
+                    } else {
-                        "dossierTemplateId",
+                        BeanUtils.copyProperties(typeValueRequest,
-                        "dossierId",
+                                                 type,
-                        "entries",
+                                                 "type",
-                        "falsePositiveEntries",
+                                                 "dossierTemplateId",
-                        "falseRecommendationEntries",
+                                                 "dossierId",
-                        "dossierTemplate",
+                                                 "entries",
-                        "dossier",
+                                                 "falsePositiveEntries",
-                        "id",
+                                                 "falseRecommendationEntries",
-                        "version",
+                                                 "dossierTemplate",
-                        "dossierDictionaryOnly");
+                                                 "dossier",
-            }
+                                                 "id",
-            typeRepository.save(type);
+                                                 "version",
-        });
+                                                 "dossierDictionaryOnly");
+                    }
+                    typeRepository.saveAndFlush(type);
+                });
     }
 
 
@@ -267,10 +274,29 @@ public class DictionaryPersistenceService {
     }
 
 
+    public void saveType(TypeEntity type) {
+
+        typeRepository.saveAndFlush(type);
+    }
+
+
     @Transactional
     public int undeleteType(String typeId) {
 
         return typeRepository.unSoftDeleteTypeById(typeId);
     }
 
+
+    public List<TypeRankSummary> getTypeRankSummaryList(String dossierTemplateId) {
+
+        return typeRepository.findTypeRankSummaryList(dossierTemplateId);
+
+    }
+
+    @Transactional
+    public void updateRankForType(String typeId, int newRank) {
+
+        typeRepository.updateRankForType(typeId, newRank);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java

@@ -116,23 +116,28 @@ public class DossierPersistenceService {
     @Transactional
     public void update(String dossierId, CreateOrUpdateDossierRequest createOrUpdateDossierRequest) {
 
-        dossierRepository.findById(dossierId).ifPresent(dossier -> {
+        dossierRepository.findById(dossierId)
-            BeanUtils.copyProperties(createOrUpdateDossierRequest, dossier, "watermarkId", "previewWatermarkId");
+                .ifPresent(dossier -> {
-            dossier.setDossierTemplate(dossierTemplateRepository.getOne(createOrUpdateDossierRequest.getDossierTemplateId()));
+                    BeanUtils.copyProperties(createOrUpdateDossierRequest, dossier, "watermarkId", "previewWatermarkId");
-            dossier.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+                    dossier.setDossierTemplate(dossierTemplateRepository.getOne(createOrUpdateDossierRequest.getDossierTemplateId()));
+                    dossier.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
 
-            dossier.getReportTemplates()
+                    dossier.getReportTemplates()
-                    .forEach(t -> t.getDossiers().removeIf(d -> d.getId().equals(dossierId) && !createOrUpdateDossierRequest.getReportTemplateIds().contains(t.getTemplateId())));
+                            .forEach(t -> t.getDossiers()
-            var reportTemplates = reportTemplateRepository.findAllById(createOrUpdateDossierRequest.getReportTemplateIds());
+                                    .removeIf(d -> d.getId().equals(dossierId) && !createOrUpdateDossierRequest.getReportTemplateIds().contains(t.getTemplateId())));
-            reportTemplates.forEach(r -> {
+                    var reportTemplates = reportTemplateRepository.findAllById(createOrUpdateDossierRequest.getReportTemplateIds());
-                if (!r.getDossiers().stream().map(DossierEntity::getId).collect(Collectors.toSet()).contains(dossierId)) {
+                    reportTemplates.forEach(r -> {
-                    r.getDossiers().add(dossier);
+                        if (!r.getDossiers()
-                }
+                                .stream()
-            });
+                                .map(DossierEntity::getId)
-            dossier.setReportTemplates(reportTemplates);
+                                .collect(Collectors.toSet()).contains(dossierId)) {
-            this.handleDossierStatus(createOrUpdateDossierRequest, dossier);
+                            r.getDossiers().add(dossier);
-            this.handleWatermark(createOrUpdateDossierRequest, dossier);
+                        }
-        });
+                    });
+                    dossier.setReportTemplates(reportTemplates);
+                    this.handleDossierStatus(createOrUpdateDossierRequest, dossier);
+                    this.handleWatermark(createOrUpdateDossierRequest, dossier);
+                });
 
     }
 
@@ -151,7 +156,8 @@ public class DossierPersistenceService {
 
     public DossierEntity findByDossierId(String dossierId) {
 
-        return dossierRepository.findById(dossierId).orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
+        return dossierRepository.findById(dossierId)
+                .orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
     }
 
 
@@ -250,4 +256,10 @@ public class DossierPersistenceService {
         }
     }
 
+
+    public String getDossierTemplateId(String dossierId) {
+
+        return dossierRepository.findDossierTemplateId(dossierId);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierTemplatePersistenceService.java

@@ -5,9 +5,9 @@ import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
-
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
@@ -35,6 +35,7 @@ public class DossierTemplatePersistenceService {
 
     public static final String DOSSIER_TEMPLATE_NOT_FOUND_MESSAGE = "DossierTemplate with Id %s not found.";
 
+    private final DictionaryPersistenceService dictionaryPersistenceService;
     private final DossierTemplateRepository dossierTemplateRepository;
     private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
     private final RulesPersistenceService rulesPersistenceService;
@@ -225,4 +226,19 @@ public class DossierTemplatePersistenceService {
         dossierTemplateRepository.saveAndFlush(dossierTemplate);
     }
 
+
+    public void validateDossierTemplateForDuplicateRanks(String dossierTemplateId) {
+
+        var duplicateRanks = dictionaryPersistenceService.getTypeRankSummaryList(dossierTemplateId)
+                .stream()
+                .filter(t -> t.getTypesCount() > 1)
+                .collect(Collectors.toList());
+        if (!duplicateRanks.isEmpty()) {
+            String errorMessage = "Duplicate ranks found in dossier template " + dossierTemplateId + "\n" + duplicateRanks.stream()
+                    .map(t -> String.format(" Rank %d has %d entries", t.getRank(), t.getTypesCount()))
+                    .collect(Collectors.joining("\n"));
+            throw new BadRequestException(errorMessage);
+        }
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java

@@ -1,5 +1,9 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
 
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository;
@@ -8,10 +12,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-
 @Service
 @RequiredArgsConstructor
 public class SaasMigrationStatusPersistenceService {
@@ -41,6 +41,11 @@ public class SaasMigrationStatusPersistenceService {
         return migrationStatusOptional.isPresent() && migrationStatusOptional.get().getStatus() != SaasMigrationStatus.FINISHED;
     }
 
+    public boolean migrationFinishedForTenant() {
+
+        return saasMigrationStatusRepository.findAllWhereStatusNotFinishedAndNotError() == 0;
+    }
+
 
     @Transactional
     public void createMigrationRequiredStatus(String dossierId, String fileId) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java

@@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ManualRedactionRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
@@ -47,6 +48,13 @@ public class AddRedactionPersistenceService {
     }
 
 
+    @Transactional
+    public List<ManualRedactionEntryEntity> insert(List<ManualRedactionEntryEntity> manualRedactionEntryEntities) {
+
+        return manualRedactionRepository.saveAllAndFlush(manualRedactionEntryEntities);
+    }
+
+
     public Optional<ManualRedactionEntryEntity> findById(String annotationId, String fileId) {
 
         return manualRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId));
@@ -72,9 +80,10 @@ public class AddRedactionPersistenceService {
     }
 
 
-    public List<ManualRedactionEntryEntity> findAddRedactions(String fileId, boolean includeDeletions) {
+    @Transactional
+    public List<ManualRedactionEntryEntity> findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) {
 
-        return manualRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions);
+        return manualRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges());
     }
 
 
@@ -84,12 +93,6 @@ public class AddRedactionPersistenceService {
     }
 
 
-    public List<ManualRedactionEntryEntity> findUnprocessedRedactions(String fileId) {
-
-        return manualRedactionRepository.findByFileIdAndUnprocessed(fileId);
-    }
-
-
     @Transactional
     public void hardDelete(String fileId, String annotationId) {
 
@@ -110,8 +113,9 @@ public class AddRedactionPersistenceService {
         manualRedactionRepository.updateSoftDelete(new AnnotationEntityId(annotationId, fileId), null);
     }
 
+
     @Transactional
-    public void update(ManualRedactionEntryEntity manualRedactionEntry) {
+    public void updateOrCreate(ManualRedactionEntryEntity manualRedactionEntry) {
 
         manualRedactionRepository.saveAndFlush(manualRedactionEntry);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.springframework.beans.BeanUtils;
@@ -12,6 +13,7 @@ import org.springframework.transaction.annotation.Transactional;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ForceRedactionRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
 
@@ -35,6 +37,10 @@ public class ForceRedactionPersistenceService {
         forceRedactionRepository.saveAndFlush(manualForceRedaction);
     }
 
+    public List<ManualForceRedactionEntity> findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) {
+
+        return forceRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed());
+    }
 
     @Transactional
     public void hardDelete(String fileId, String annotationId) {
@@ -64,13 +70,13 @@ public class ForceRedactionPersistenceService {
     }
 
 
-    public Set<ManualForceRedactionEntity> findForceRedactions(String fileId, boolean includeDeletions) {
+    public Set<ManualForceRedactionEntity> findForceRedactions(String fileId, boolean includeDeletions, boolean includeDictChanges) {
 
         return new HashSet<>(forceRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions));
     }
 
 
-    public Set<ManualForceRedactionEntity> findUnprocessedForceRedactions(String fileId) {
+    public Set<ManualForceRedactionEntity> findUnprocessedForceRedactions(String fileId, boolean includeDictChanges) {
 
         return new HashSet<>(forceRedactionRepository.findByFileIdAndUnprocessed(fileId));
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
@@ -14,12 +15,14 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.LegalBasisChangeRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+@Deprecated(forRemoval = true)
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -78,7 +81,7 @@ public class LegalBasisChangePersistenceService {
     }
 
 
-    public Set<ManualLegalBasisChangeEntity> findLegalBasisChanges(String fileId, boolean includeDeletions) {
+    public Set<ManualLegalBasisChangeEntity> findLegalBasisChanges(String fileId, boolean includeDeletions, boolean includeDictChanges) {
 
         return new HashSet<>(legalBasisChangeRepository.findByFileIdIncludeDeletions(fileId, includeDeletions));
     }
@@ -95,4 +98,9 @@ public class LegalBasisChangePersistenceService {
         legalBasisChangeRepository.markAsProcessed(new AnnotationEntityId(annotationId, fileId), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
     }
 
+    public List<ManualLegalBasisChangeEntity> findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) {
+
+        return legalBasisChangeRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed());
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RecategorizationPersistenceService.java

@@ -5,13 +5,16 @@ import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRecategorizationEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RecategorizationRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
 
@@ -23,26 +26,34 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RecategorizationPersistenceService {
 
+    private final int SECTION_MAX_LENGTH = 1024;
+
     private final RecategorizationRepository recategorizationRepository;
 
 
     public void insert(String fileId, RecategorizationRequest recategorizationRequest) {
 
-        ManualRecategorizationEntity manualImageRecategorization = new ManualRecategorizationEntity();
+        ManualRecategorizationEntity manualRecategorization = new ManualRecategorizationEntity();
-        manualImageRecategorization.setId(new AnnotationEntityId(recategorizationRequest.getAnnotationId(), fileId));
+        manualRecategorization.setId(new AnnotationEntityId(recategorizationRequest.getAnnotationId(), fileId));
-        BeanUtils.copyProperties(recategorizationRequest, manualImageRecategorization);
+        checkSection(recategorizationRequest.getSection());
-        manualImageRecategorization.setRequestDate(OffsetDateTime.now());
+        BeanUtils.copyProperties(recategorizationRequest, manualRecategorization);
-        manualImageRecategorization.setTypeId(recategorizationRequest.getDossierTemplateTypeId());
+        manualRecategorization.setRequestDate(OffsetDateTime.now());
-        recategorizationRepository.saveAndFlush(manualImageRecategorization);
+        manualRecategorization.setTypeId(recategorizationRequest.getDossierTemplateTypeId());
+        recategorizationRepository.saveAndFlush(manualRecategorization);
 
     }
 
 
+    private void checkSection(String section) {
+
+        if (!StringUtils.isEmpty(section) && section.length() > SECTION_MAX_LENGTH) {
+            throw new BadRequestException(String.format("The section is too long (%s), max length %s", section.length(), SECTION_MAX_LENGTH));
+        }
+    }
+
 
     @Transactional
-    public void updateModifiedDictionaries(String fileId,
+    public void updateModifiedDictionaries(String fileId, String annotationId, Set<String> typeIdsOfDictionaryWithAdd, Set<String> typeIdsOfDictionaryWithDelete) {
-                                           String annotationId, Set<String> typeIdsOfDictionaryWithAdd,
-                                           Set<String> typeIdsOfDictionaryWithDelete) {
 
         ManualRecategorizationEntity addRedaction = recategorizationRepository.findById(new AnnotationEntityId(annotationId, fileId))
                 .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
@@ -82,13 +93,19 @@ public class RecategorizationPersistenceService {
     }
 
 
-    public List<ManualRecategorizationEntity> findRecategorizations(String fileId, boolean includeDeletions) {
+    public List<ManualRecategorizationEntity> findRecategorizations(String fileId, boolean includeDeletions, boolean includeDictChanges) {
 
         return recategorizationRepository.findByFileIdIncludeDeletions(fileId, includeDeletions);
     }
 
 
-    public List<ManualRecategorizationEntity> findUnprocessedRecategorizations(String fileId) {
+    public List<ManualRecategorizationEntity> findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) {
+
+        return recategorizationRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges());
+    }
+
+
+    public List<ManualRecategorizationEntity> findUnprocessedRecategorizations(String fileId, boolean includeDictChanges) {
 
         return recategorizationRepository.findUnprocessedByFileId(fileId);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java

@@ -13,6 +13,7 @@ import org.springframework.transaction.annotation.Transactional;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -47,13 +48,13 @@ public class RemoveRedactionPersistenceService {
     }
 
 
-    public Set<IdRemovalEntity> findRemoveRedactions(String fileId, boolean includeDeletions) {
+    public Set<IdRemovalEntity> findRemoveRedactions(String fileId, boolean includeDeletions, boolean includeDictChanges) {
 
         return new HashSet<>(removeRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions));
     }
 
 
-    public Set<IdRemovalEntity> findUnprocessedRemoveRedactions(String fileId) {
+    public Set<IdRemovalEntity> findUnprocessedRemoveRedactions(String fileId, boolean includeDictChanges) {
 
         return new HashSet<>(removeRedactionRepository.findByFileIdAndUnprocessed(fileId));
     }
@@ -65,6 +66,12 @@ public class RemoveRedactionPersistenceService {
     }
 
 
+    public List<IdRemovalEntity> findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) {
+
+        return removeRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges());
+    }
+
+
     @Transactional
     public void hardDelete(String fileId, String annotationId) {
 
@@ -105,4 +112,11 @@ public class RemoveRedactionPersistenceService {
         removeRedactionRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
     }
 
+
+    public void updateOrCreate(IdRemovalEntity unprocessedManualRemove) {
+
+        removeRedactionRepository.saveAndFlush(unprocessedManualRemove);
+
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java

@@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
@@ -86,13 +87,14 @@ public class ResizeRedactionPersistenceService {
                 .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
     }
 
+
     public Optional<ManualResizeRedactionEntity> findResizeRedactionById(String fileId, String annotationId) {
 
         return resizeRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId));
     }
 
 
-    public List<ManualResizeRedactionEntity> findResizeRedactions(String fileId, boolean includeDeletions) {
+    public List<ManualResizeRedactionEntity> findResizeRedactions(String fileId, boolean includeDeletions, boolean includeDictChanges) {
 
         return resizeRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions);
     }
@@ -115,4 +117,16 @@ public class ResizeRedactionPersistenceService {
         resizeRedactionRepository.markAsProcessed(new AnnotationEntityId(annotationId, fileId), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
     }
 
+
+    public List<ManualResizeRedactionEntity> findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) {
+
+        return resizeRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges());
+    }
+
+
+    public void updateOrCreate(ManualResizeRedactionEntity unprocessedManualResize) {
+
+        resizeRedactionRepository.saveAndFlush(unprocessedManualResize);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java

@@ -94,4 +94,8 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
     @Query("select count(d) from DossierEntity d where d.watermarkId = :watermarkId or d.previewWatermarkId = :watermarkId")
     int countDossiersWithWatermarkInUse(@Param("watermarkId") long watermarkId);
 
+
+    @Query("select d.dossierTemplateId from DossierEntity d where d.id = :dossierId")
+    String findDossierTemplateId(@Param("dossierId") String dossierId);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -31,7 +31,11 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying
-    @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + " e.lastUpdated = :lastUpdated " + " where e.id =:fileId")
+    @Query("update FileEntity e set e.hasRedactions = :hasRedactions ,"
+           + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions,"
+           + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, "
+           + " e.lastUpdated = :lastUpdated "
+           + " where e.id =:fileId")
     void updateFlags(@Param("fileId") String fileId,
                      @Param("lastUpdated") OffsetDateTime lastUpdated,
                      @Param("hasRedactions") boolean hasRedactions,
@@ -43,7 +47,12 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying
-    @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
+    @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, "
+           + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, "
+           + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, "
+           + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, "
+           + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter "
+           + "where f.id = :fileId")
     void updateProcessingStatus(@Param("fileId") String fileId,
                                 @Param("numberOfPages") int numberOfPages,
                                 @Param("processingStatus") ProcessingStatus processingStatus,
@@ -61,7 +70,8 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying
-    @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId")
+    @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate,"
+           + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId")
     void updateWorkflowStatus(@Param("fileId") String fileId,
                               @Param("workflowStatus") WorkflowStatus workflowStatus,
                               @Param("lastUpdated") OffsetDateTime lastUpdated,
@@ -83,7 +93,9 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + " f.hasHighlights = :hasHighlights, f.processingErrorCounter = :processingErrorCounter " + " where f.id = :fileId")
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated,"
+           + " f.hasHighlights = :hasHighlights, f.processingErrorCounter = :processingErrorCounter "
+           + " where f.id = :fileId")
     void updateProcessingStatus(@Param("fileId") String fileId,
                                 @Param("processingStatus") ProcessingStatus processingStatus,
                                 @Param("lastUpdated") OffsetDateTime lastUpdated,
@@ -92,7 +104,8 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter "
+           + "where f.id = :fileId")
     void updateProcessingStatus(@Param("fileId") String fileId,
                                 @Param("processingStatus") ProcessingStatus processingStatus,
                                 @Param("lastUpdated") OffsetDateTime lastUpdated,
@@ -109,7 +122,8 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying(clearAutomatically = true, flushAutomatically = true)
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter "
+           + "where f.id = :fileId")
     void updateProcessingStatus(@Param("fileId") String fileId,
                                 @Param("processingStatus") ProcessingStatus processingStatus,
                                 @Param("lastUpdated") OffsetDateTime lastUpdated,
@@ -118,7 +132,8 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId")
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated,"
+           + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId")
     void setUpdateStatusIndexingSuccessful(@Param("fileId") String fileId,
                                            @Param("processingStatus") ProcessingStatus processingStatus,
                                            @Param("lastUpdated") OffsetDateTime lastUpdated,
@@ -155,7 +170,13 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.hardDeletedTime = :hardDeletedTime, " + "f.deleted = case " + "   when f.deleted is null then :deleted " + "   when f.deleted is not null then f.deleted " + "end " + "where f.id = :fileId")
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated,"
+           + "f.hardDeletedTime = :hardDeletedTime, "
+           + "f.deleted = case "
+           + "   when f.deleted is null then :deleted "
+           + "   when f.deleted is not null then f.deleted "
+           + "end "
+           + "where f.id = :fileId")
     int setHardDelete(@Param("fileId") String fileId,
                       @Param("processingStatus") ProcessingStatus processingStatus,
                       @Param("lastUpdated") OffsetDateTime lastUpdated,
@@ -185,7 +206,18 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " + "f.fileManipulationDate = :lastUploaded, " + "f.ocrEndTime = null, f.ocrStartTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, " + "f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, " + "f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, " + "f.processingErrorCounter = 0, f.errorCause = null, f.errorQueue = null, f.errorService = null, " + "f.errorTimestamp = null where f.id = :fileId")
+    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, "
+           + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, "
+           + "f.fileManipulationDate = :lastUploaded, "
+           + "f.ocrEndTime = null, f.ocrStartTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, "
+           + "f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, "
+           + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, "
+           + "f.redactionModificationDate = null, "
+           + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
+           + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
+           + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, "
+           + "f.processingErrorCounter = 0, f.errorCause = null, f.errorQueue = null, f.errorService = null, "
+           + "f.errorTimestamp = null where f.id = :fileId")
     int overwriteFile(@Param("fileId") String fileId,
                       @Param("filename") String filename,
                       @Param("uploader") String uploader,
@@ -196,7 +228,16 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastProcessed = null," + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0, " + "f.ocrStartTime = null, f.ocrEndTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, " + "f.errorCause = null, f.errorQueue = null, f.errorService = null, f.errorTimestamp = null " + "where f.id = :fileId")
+    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, "
+           + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, "
+           + "f.lastProcessed = null,"
+           + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
+           + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
+           + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
+           + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0, "
+           + "f.ocrStartTime = null, f.ocrEndTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, "
+           + "f.errorCause = null, f.errorQueue = null, f.errorService = null, f.errorTimestamp = null "
+           + "where f.id = :fileId")
     int overwriteFileAndKeepManualRedactions(@Param("fileId") String fileId,
                                              @Param("filename") String filename,
                                              @Param("uploader") String uploader,
@@ -205,11 +246,17 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
                                              @Param("lastUpdated") OffsetDateTime lastUpdated);
 
 
-    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId " + "where d.dossierTemplateId = :dossierTemplateId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
+    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId "
+           + "where d.dossierTemplateId = :dossierTemplateId"
+           + " and ((f.deleted is not null and f.hardDeletedTime is null) or "
+           + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
     int countSoftDeletedFilesPerDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
 
 
-    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId " + "where d.id = :dossierId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null))")
+    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId "
+           + "where d.id = :dossierId"
+           + " and ((f.deleted is not null and f.hardDeletedTime is null) or "
+           + " (d.softDeletedTime is not null and d.hardDeletedTime is null))")
     int countSoftDeletedFilesPerDossierId(@Param("dossierId") String dossierId);
 
 
@@ -238,7 +285,11 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
                                     @Param("lastUpdated") OffsetDateTime lastUpdated);
 
 
-    @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId " + "where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )" + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries")
+    @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId "
+           + "where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false "
+           + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )"
+           + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null "
+           + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries")
     List<FileEntity> getAllRelevantStatusesForReanalysisScheduler(@Param("maxRetries") int maxRetries);
 
 
@@ -251,25 +302,45 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     List<FileEntity> getSoftDeletedFiles(@Param("dossierIds") List<String> dossierIds);
 
 
-    @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.processingStatus ")
+    @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f "
+           + "inner join DossierEntity d on d.id = f.dossierId "
+           + "where f.deleted is null and f.hardDeletedTime is null "
+           + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null "
+           + "and d.dossierTemplateId = :dossierTemplateId "
+           + "group by f.processingStatus ")
     List<FileProcessingStatusProjection> countFilesByProcessingStatus(@Param("dossierTemplateId") String dossierTemplateId);
 
 
-    @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.workflowStatus ")
+    @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f "
+           + "inner join DossierEntity d on d.id = f.dossierId "
+           + "where f.deleted is null and f.hardDeletedTime is null "
+           + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null "
+           + "and d.dossierTemplateId = :dossierTemplateId "
+           + "group by f.workflowStatus ")
     List<FileWorkflowStatusProjection> countFilesByWorkflowStatus(@Param("dossierTemplateId") String dossierTemplateId);
 
 
-    @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, " + "COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + "from file join dossier on file.dossier_id = dossier.id " + "where file.deleted is null and file.hard_deleted_time is null " + "and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true)
+    @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, "
+                   + "COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages "
+                   + "from file join dossier on file.dossier_id = dossier.id "
+                   + "where file.deleted is null and file.hard_deleted_time is null "
+                   + "and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null"
+                   + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true)
     FilePageCountsProjection countPages(@Param("dossierTemplateId") String dossierTemplateId);
 
 
-    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + "f.hardDeletedTime is null and f.deleted is null and " + "d.dossierTemplateId = :dossierTemplateId and " + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null")
+    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where "
+           + "f.hardDeletedTime is null and f.deleted is null and "
+           + "d.dossierTemplateId = :dossierTemplateId and "
+           + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null")
     int countActiveFiles(@Param("dossierTemplateId") String dossierTemplateId);
 
 
     @Transactional
     @Modifying(clearAutomatically = true)
-    @Query(value = "update FileEntity f set f.numberOfOCRedPages = :numberOfOCRedPages, " + "f.numberOfPagesToOCR = :numberOfPagesToOCR, f.ocrEndTime = :ocrEndTime, " + "f.lastUpdated = :lastUpdated where f.id = :fileId")
+    @Query(value = "update FileEntity f set f.numberOfOCRedPages = :numberOfOCRedPages, "
+                   + "f.numberOfPagesToOCR = :numberOfPagesToOCR, f.ocrEndTime = :ocrEndTime, "
+                   + "f.lastUpdated = :lastUpdated where f.id = :fileId")
     void updateOCRStatus(@Param("fileId") String fileId,
                          @Param("numberOfPagesToOCR") int numberOfPagesToOCR,
                          @Param("numberOfOCRedPages") int numberOfOCRedPages,
@@ -300,11 +371,11 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
      * @return a List of Tuples of fileIds and dossierIds, where a flag calculation is necessary
      */
     @Query("""
-            select distinct f.id, f.dossierId
+                select distinct f.id, f.dossierId
-            from FileEntity f
+                from FileEntity f
-            left join ViewedPageEntity v on f.id = v.file.id and v.id.userId = f.assignee
+                left join ViewedPageEntity v on f.id = v.file.id and v.id.userId = f.assignee
-            where (f.lastFlagCalculation is NULL and f.lastProcessed is not NULL) or f.lastManualChangeDate > f.lastFlagCalculation or f.lastProcessed > f.lastFlagCalculation or f.lastFlagCalculation < v.viewedTime
+                where (f.lastFlagCalculation is NULL and f.lastProcessed is not NULL) or f.lastManualChangeDate > f.lastFlagCalculation or f.lastProcessed > f.lastFlagCalculation or f.lastFlagCalculation < v.viewedTime
-        """)
+            """)
     List<Tuple> getFileIdentifiersWhereAnalysisFlagCalculationIsRequired();
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/SaasMigrationStatusRepository.java

@@ -32,4 +32,8 @@ public interface SaasMigrationStatusRepository extends JpaRepository<SaasMigrati
     @Query("select count(*) from SaasMigrationStatusEntity")
     int countAll();
 
+
+    @Query("select count(*) from SaasMigrationStatusEntity e where e.status != 'FINISHED' and e.status != 'ERROR'")
+    int findAllWhereStatusNotFinishedAndNotError();
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java

@@ -11,11 +11,18 @@ import org.springframework.data.repository.query.Param;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionarySummaryResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.TypeRankSummary;
 
 public interface TypeRepository extends JpaRepository<TypeEntity, String> {
 
-    @Query("select t from TypeEntity t where t.dossierTemplateId = :dossierTemplate and t.dossierId = :dossier and t.rank = :rank  and t.softDeletedTime is null")
+    @Query("select t from TypeEntity t where t.dossierTemplateId = :dossierTemplateId and "
-    Optional<TypeEntity> findOneByDossierTemplateIdAndDossierIdAndRank(@Param("dossierTemplate") String dossierTemplate, @Param("dossier") String dossier, @Param("rank") int rank);
+           + "(CASE "
+           + "  WHEN :dossierId is null THEN t.dossierId is null "
+           + "  WHEN :dossierId is not null THEN t.dossierId = :dossierId "
+           + "END) and t.rank = :rank  and t.softDeletedTime is null")
+    List<TypeEntity> findByDossierTemplateIdAndDossierIdAndRank(@Param("dossierTemplateId") String dossierTemplateId,
+                                                                @Param("dossierId") String dossierId,
+                                                                @Param("rank") int rank);
 
 
     List<TypeEntity> findByDossierId(String dossierId);
@@ -24,6 +31,7 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
     @Query("select t from TypeEntity t where t.id = :typeId and t.softDeletedTime is null")
     Optional<TypeEntity> findByIdAndNotDeleted(@Param("typeId") String typeId);
 
+
     @Modifying
     @Query("update TypeEntity t set t.version = t.version + 1 where t.id = :typeId")
     void updateByIdSetIncrementVersionByOne(@Param("typeId") String typeId);
@@ -47,21 +55,37 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
     @Query("select coalesce(sum(t.version), 0) from TypeEntity t where t.dossierTemplateId = :dossierTemplateId and t.dossierId is null")
     long getVersionForDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
 
+
     @Query("""
-        select new com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionarySummaryResponse(dt.id, t.id, t.type, t.label, count(e))
+            select new com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionarySummaryResponse(dt.id, t.id, t.type, t.label, count(e))
-        from DossierTemplateEntity dt
+            from DossierTemplateEntity dt
-        left join TypeEntity t on t.dossierTemplateId = dt.id
+            left join TypeEntity t on t.dossierTemplateId = dt.id
-        left join DictionaryEntryEntity e on e.typeId = t.id
+            left join DictionaryEntryEntity e on e.typeId = t.id
-        where t.softDeletedTime is null and dt.id in :dossierTemplateIds and t.dossierId is null and (e.entryId is null or e.deleted = false)
+            where t.softDeletedTime is null and dt.id in :dossierTemplateIds and t.dossierId is null and (e.entryId is null or e.deleted = false)
-        group by dt.id, t.id, t.type, t.label""")
+            group by dt.id, t.id, t.type, t.label""")
     List<DictionarySummaryResponse> findDictionarySummaryList(@Param("dossierTemplateIds") Set<String> dossierTemplateIds);
 
+
+    @Query("""
+            select new com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.TypeRankSummary(t.dossierTemplateId, t.dossierId, t.rank, count(t))
+            from TypeEntity t where t.dossierTemplateId = :dossierTemplateId and t.softDeletedTime is null
+            group by t.dossierTemplateId, t.dossierId, t.rank
+            order by t.rank""")
+    List<TypeRankSummary> findTypeRankSummaryList(@Param("dossierTemplateId") String dossierTemplateId);
+
+
     @Modifying(clearAutomatically = true, flushAutomatically = true)
     @Query("Update TypeEntity t set t.softDeletedTime = offset_datetime where t.id = :typeId")
     void softDeleteTypeById(@Param("typeId") String typeId);
 
+
     @Modifying
     @Query("Update TypeEntity t set t.softDeletedTime = null where t.id = :typeId and t.softDeletedTime is not null")
     int unSoftDeleteTypeById(@Param("typeId") String typeId);
 
+
+    @Modifying
+    @Query("Update TypeEntity t set t.rank = :newRank, t.version = t.version + 1 where t.id = :typeId")
+    void updateRankForType(@Param("typeId") String typeId, @Param("newRank") int newRank);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ForceRedactionRepository.java

@@ -35,4 +35,14 @@ public interface ForceRedactionRepository extends JpaRepository<ManualForceRedac
     @Query("update ManualForceRedactionEntity mfr set mfr.processedDate = :processedDate where mfr.id = :annotationEntityId")
     void markAsProcessed(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("processedDate") OffsetDateTime processedDate);
 
+    @Query("""
+            select m
+            from ManualForceRedactionEntity m
+            where m.id.fileId = :fileId
+            and ((:unprocessed = true and m.processedDate is null) or :unprocessed = false)
+            and (:includeDeletions = true or m.softDeletedTime is null)
+            """)
+    List<ManualForceRedactionEntity> findByFileIdAndOptions(@Param("fileId") String fileId,
+                                                              @Param("includeDeletions") boolean includeDeletions,
+                                                              @Param("unprocessed") boolean unprocessed);
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/LegalBasisChangeRepository.java

@@ -12,6 +12,7 @@ import org.springframework.data.repository.query.Param;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity;
 
+@Deprecated(forRemoval = true)
 public interface LegalBasisChangeRepository extends JpaRepository<ManualLegalBasisChangeEntity, AnnotationEntityId>, AnnotationEntityRepository {
 
     @Modifying
@@ -35,4 +36,16 @@ public interface LegalBasisChangeRepository extends JpaRepository<ManualLegalBas
     @Query("update ManualLegalBasisChangeEntity mlbc set mlbc.processedDate = :processedDate where mlbc.id = :annotationEntityId")
     void markAsProcessed(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("processedDate") OffsetDateTime processedDate);
 
+
+    @Query("""
+            select m
+            from ManualLegalBasisChangeEntity m
+            where m.id.fileId = :fileId
+            and ((:unprocessed = true and m.processedDate is null) or :unprocessed = false)
+            and (:includeDeletions = true or m.softDeletedTime is null)
+            """)
+    List<ManualLegalBasisChangeEntity> findByFileIdAndOptions(@Param("fileId") String fileId,
+                                                              @Param("includeDeletions") boolean includeDeletions,
+                                                              @Param("unprocessed") boolean unprocessed);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java

@@ -18,6 +18,7 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
     @Query("update ManualRedactionEntryEntity m set m.softDeletedTime = :softDeleteTime where m.id = :id")
     void updateSoftDelete(@Param("id") AnnotationEntityId id, @Param("softDeleteTime") OffsetDateTime softDeleteTime);
 
+
     @Modifying
     @Query("update ManualRedactionEntryEntity m set m.addToDictionary = :isAddOrRemoveFromDictionary, m.addToDossierDictionary = :isAddOrRemoveFromDossierDictionary where m.id = :id")
     void updateStatus(@Param("id") AnnotationEntityId id,
@@ -37,6 +38,19 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
     List<ManualRedactionEntryEntity> findByFileIdAndUnprocessed(@Param("fileId") String fileId);
 
 
+    @Query("""
+            select m
+            from ManualRedactionEntryEntity m
+            where m.id.fileId = :fileId
+            and ((:unprocessed = true and m.processedDate is null) or :unprocessed = false)
+            and (:includeDeletions = true or m.softDeletedTime is null)
+            and (:includeDictChanges = true or (m.addToDictionary = false and m.addToAllDossiers = false))
+            """)
+    List<ManualRedactionEntryEntity> findByFileIdAndOptions(@Param("fileId") String fileId,
+                                                            @Param("includeDeletions") boolean includeDeletions,
+                                                            @Param("unprocessed") boolean unprocessed,
+                                                            @Param("includeDictChanges") boolean includeDictChanges);
+
     @Modifying
     @Query("update ManualRedactionEntryEntity m set m.processedDate = :processedDate where m.id = :annotationEntityId")
     void markAsProcessed(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("processedDate") OffsetDateTime processedDate);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RecategorizationRepository.java

@@ -18,17 +18,35 @@ public interface RecategorizationRepository extends JpaRepository<ManualRecatego
     @Query("update ManualRecategorizationEntity mir set mir.softDeletedTime = :softDeletedTime where mir.id = :annotationEntityId")
     void updateSoftDelete(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("softDeletedTime") OffsetDateTime softDeletedTime);
 
+
     @Query("select mir from ManualRecategorizationEntity mir where mir.id = :annotationEntityId and mir.softDeletedTime is null")
     Optional<ManualRecategorizationEntity> findByIdAndNotSoftDeleted(@Param("annotationEntityId") AnnotationEntityId annotationEntityId);
 
+
     @Query("select mir from ManualRecategorizationEntity mir where mir.id.fileId = :fileId and (:includeDeletions = true or mir.softDeletedTime is null)")
     List<ManualRecategorizationEntity> findByFileIdIncludeDeletions(@Param("fileId") String fileId, @Param("includeDeletions") boolean includeDeletions);
 
+
     @Query("select mir from ManualRecategorizationEntity mir where mir.id.fileId = :fileId and mir.processedDate is null")
     List<ManualRecategorizationEntity> findUnprocessedByFileId(@Param("fileId") String fileId);
 
+
     @Modifying
     @Query("update ManualRecategorizationEntity mir set mir.processedDate = :processedDate where mir.id = :annotationEntityId")
     void markAsProcessed(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("processedDate") OffsetDateTime processedDate);
 
+
+    @Query("""
+            select m
+            from ManualRecategorizationEntity m
+            where m.id.fileId = :fileId
+            and ((:unprocessed = true and m.processedDate is null) or :unprocessed = false)
+            and (:includeDeletions = true or m.softDeletedTime is null)
+            and (:includeDictChanges = true or (m.addToDictionary = false and m.addToAllDossiers = false))
+            """)
+    List<ManualRecategorizationEntity> findByFileIdAndOptions(@Param("fileId") String fileId,
+                                                              @Param("includeDeletions") boolean includeDeletions,
+                                                              @Param("unprocessed") boolean unprocessed,
+                                                              @Param("includeDictChanges") boolean includeDictChanges);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java

@@ -32,4 +32,16 @@ public interface RemoveRedactionRepository extends JpaRepository<IdRemovalEntity
     void markAsProcessed(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("processedDate") OffsetDateTime processedDate);
 
 
+    @Query("""
+            select m
+            from IdRemovalEntity m
+            where m.id.fileId = :fileId
+            and ((:unprocessed = true and m.processedDate is null) or :unprocessed = false)
+            and (:includeDeletions = true or m.softDeletedTime is null)
+            and (:includeDictChanges = true or (m.removeFromDictionary = false and m.removeFromAllDossiers = false))
+            """)
+    List<IdRemovalEntity> findByFileIdAndOptions(@Param("fileId") String fileId,
+                                                              @Param("includeDeletions") boolean includeDeletions,
+                                                              @Param("unprocessed") boolean unprocessed,
+                                                              @Param("includeDictChanges") boolean includeDictChanges);
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ResizeRedactionRepository.java

@@ -43,4 +43,16 @@ public interface ResizeRedactionRepository extends JpaRepository<ManualResizeRed
     @Query("update ManualResizeRedactionEntity mir set mir.processedDate = :processedDate where mir.id = :annotationEntityId")
     void markAsProcessed(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("processedDate") OffsetDateTime processedDate);
 
+    @Query("""
+            select m
+            from ManualResizeRedactionEntity m
+            where m.id.fileId = :fileId
+            and ((:unprocessed = true and m.processedDate is null) or :unprocessed = false)
+            and (:includeDeletions = true or m.softDeletedTime is null)
+            and (:includeDictChanges = true or (m.updateDictionary = false and m.addToAllDossiers = false))
+            """)
+    List<ManualResizeRedactionEntity> findByFileIdAndOptions(@Param("fileId") String fileId,
+                                                              @Param("includeDeletions") boolean includeDeletions,
+                                                              @Param("unprocessed") boolean unprocessed,
+                                                              @Param("includeDictChanges") boolean includeDictChanges);
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionAnalysisResponseReceiver.java

@@ -57,7 +57,7 @@ public class RedactionAnalysisResponseReceiver {
                 if (StringUtils.isNullOrEmpty(manualRedactionEntry.getSection()) || (!StringUtils.isNullOrEmpty(unprocessedManualEntity.getSection()) && !manualRedactionEntry.getSection().equals(unprocessedManualEntity.getSection()))) {
                     manualRedactionEntry.setSection(unprocessedManualEntity.getSection());
                 }
-                addRedactionPersistenceService.update(manualRedactionEntry);
+                addRedactionPersistenceService.updateOrCreate(manualRedactionEntry);
             }
 
             Optional<ManualResizeRedactionEntity> optionalManualResizeRedactionEntity = resizeRedactionPersistenceService.findResizeRedactionById(fileId, unprocessedManualEntity.getAnnotationId());

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/ManualRecategorizationMapper.java

@@ -5,7 +5,7 @@ import java.util.function.BiConsumer;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRecategorizationEntity;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 
-public class ManualImageRecategorizationMapper implements BiConsumer<ManualRecategorizationEntity, ManualRecategorization> {
+public class ManualRecategorizationMapper implements BiConsumer<ManualRecategorizationEntity, ManualRecategorization> {
 
     @Override
     public void accept(ManualRecategorizationEntity manualRecategorizationEntity, ManualRecategorization manualRedactionEntry) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/ManualRedactionEntryMapper.java

@@ -7,7 +7,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
 
-public class ManualRedactionMapper implements BiConsumer<ManualRedactionEntryEntity, ManualRedactionEntry> {
+public class ManualRedactionEntryMapper implements BiConsumer<ManualRedactionEntryEntity, ManualRedactionEntry> {
 
     @Override
     public void accept(ManualRedactionEntryEntity manualRedactionEntryEntity, ManualRedactionEntry manualRedactionEntry) {

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml

@@ -181,3 +181,19 @@ databaseChangeLog:
       file: db/changelog/tenant/118-delete-unapproved-manual-changes-and-drop-column-status.yaml
   - include:
       file: db/changelog/tenant/119-set-add-to-all-dossiers-correctly-in-manual-redaction-table.yaml
+  - include:
+      file: db/changelog/tenant/120-add-legal-basis-change-to-manual-recategorization.yaml
+  - include:
+      file: db/changelog/tenant/sql/205-add-dossier-dictionaries-as-entity.sql
+  - include:
+      file: db/changelog/tenant/121-set-dictionary-entry-type-for-dictionary-adds-where-null.yaml
+  - include:
+      file: db/changelog/tenant/sql/206-remove-manual-redactions-on-non-existing-pages.sql
+  - include:
+      file: db/changelog/tenant/122-add-legal-basis-variables-to-recategorize.yaml
+  - include:
+      file: db/changelog/tenant/123-add-value-to-recategorize.yaml
+  - include:
+      file: db/changelog/tenant/124-create-migration-required-status-for-each-present-file.yaml
+  - include:
+      file: db/changelog/tenant/sql/207-acl-migration-cleanup.sql

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/120-add-legal-basis-change-to-manual-recategorization.yaml

@@ -0,0 +1,18 @@
+databaseChangeLog:
+  - changeSet:
+      id: add-legal-basis-change-to-manual-recategorization
+      author: ali
+      changes:
+        - addColumn:
+            tableName: manual_recategorization
+            columns:
+              - column:
+                  name: legal_basis
+                  type: VARCHAR(4000)
+                  defaultValue: ''
+                  constraints:
+                      nullable: false
+        - modifyDataType:
+            columnName: legal_basis
+            newDataType: VARCHAR(4000)
+            tableName: manual_redaction

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/121-set-dictionary-entry-type-for-dictionary-adds-where-null.yaml

@@ -0,0 +1,13 @@
+databaseChangeLog:
+  - changeSet:
+      id: set-dictionary-entry-type-for-dictionary-adds-where-null
+      author: kilian
+
+      changes:
+        - update:
+            tableName: manual_redaction
+            columns:
+              - column:
+                  name: dictionary_entry_type
+                  value: 'ENTRY'
+            where: "add_to_dictionary AND dictionary_entry_type IS NULL"

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/122-add-legal-basis-variables-to-recategorize.yaml

@@ -0,0 +1,11 @@
+databaseChangeLog:
+  - changeSet:
+      id: add-legal-basis-variables-to-recategorize
+      author: ali
+      changes:
+        - addColumn:
+            tableName: manual_recategorization
+            columns:
+              - column:
+                  name: section
+                  type: VARCHAR(1024)

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/123-add-value-to-recategorize.yaml

@@ -0,0 +1,11 @@
+databaseChangeLog:
+  - changeSet:
+      id: add-legal-basis-variables-to-recategorize
+      author: ali
+      changes:
+        - addColumn:
+            tableName: manual_recategorization
+            columns:
+              - column:
+                  name: value
+                  type: VARCHAR(4000)

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/124-create-migration-required-status-for-each-present-file.yaml

@@ -0,0 +1,10 @@
+databaseChangeLog:
+  - changeSet:
+      id: create-migration-required-status-for-each-present-file
+      author: Kilian
+      changes:
+        - sql:
+            sql: |
+              INSERT INTO saas_migration_status (file_id, dossier_id, status)
+              SELECT id, dossier_id, 'MIGRATION_REQUIRED'
+              FROM file;

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/sql/205-add-dossier-dictionaries-as-entity.sql

@@ -0,0 +1,11 @@
+INSERT INTO entity (id, add_to_dictionary_action, description, dossier_id, dossier_template_id, hex_color, is_case_insensitive, is_hint, is_recommendation,
+                              label, rank, type, version, recommendation_hex_color, has_dictionary, system_managed, auto_hide_skipped, skipped_hex_color,
+                              dossier_dictionary_only)
+SELECT distinct trim(trailing ':' from trim(trailing e.dossier_id from e.id)), TRUE, 'Entries in this dictionary will only be redacted in this dossier', NULL, e.dossier_template_id, '#9398a0', false, false, false,
+                'Dossier Redaction', 1500, 'dossier_redaction', 1, '#8df06c', true, true, false,  '#C498FA',
+                TRUE
+FROM entity e
+WHERE (SELECT trim(trailing ':' from trim(trailing e.dossier_id from e.id))) IS NOT NULL AND (SELECT substr(e.id, POSITION(':' IN e.id) + POSITION(':' IN substr(e.id, POSITION(':' IN e.id) + 1)) + 1)) = '' IS FALSE AND NOT EXISTS (SELECT 1 FROM entity ee WHERE ee.id =
+
+                                                                                                                                                                                                                                                                               trim(trailing ':' from trim(trailing e.dossier_id from e.id))
+);

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/sql/206-remove-manual-redactions-on-non-existing-pages.sql

@@ -0,0 +1,7 @@
+Delete
+from manual_redaction
+where concat(annotation_id, file_id) in (SELECT concat(manual_redaction_entry_entity_annotation_id, id)
+                                         from file file
+                                                  JOIN manual_redaction_entry_entity_positions pos
+                                                       ON file.id = pos.manual_redaction_entry_entity_file_id
+                                         where file.number_of_pages < pos.page);

persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/sql/207-acl-migration-cleanup.sql

@@ -0,0 +1,4 @@
+ALTER TABLE acl_class DROP CONSTRAINT IF EXISTS acl_class_unique_temp;
+ALTER TABLE acl_sid DROP CONSTRAINT IF EXISTS acl_sid_unique_temp;
+ALTER TABLE acl_object_identity DROP CONSTRAINT IF EXISTS acl_object_identity_unique_temp;
+ALTER TABLE acl_entry DROP CONSTRAINT IF EXISTS acl_entry_unique_temp;

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java

@@ -50,13 +50,69 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
     private List<String> testList2 = List.of("William C. Spare", "Charalampos", "Carina Wilson", "Patricia A. Sheehy", "William c. Spare", "carlsen", "PATRICIA A. SHEEHY");
 
 
-
     @BeforeEach
     public void createDossierRedactionDictionary() {
 
         TenantContext.setTenantId("redaction");
     }
 
+
+    @Test
+    public void testAddTypesWithSameRank() {
+
+        var dossier = dossierTesterAndProvider.provideTestDossier();
+
+        var dossierTemplate = dossierTemplateClient.getDossierTemplate(dossier.getDossierTemplateId());
+
+        var returnedtype1 = dictionaryClient.addType(CreateTypeValue.builder()
+                                                             .type("test1")
+                                                             .label("test1")
+                                                             .hexColor("#fcba03")
+                                                             .rank(100)
+                                                             .hint(false)
+                                                             .caseInsensitive(false)
+                                                             .recommendation(false)
+                                                             .addToDictionaryAction(true)
+                                                             .dossierTemplateId(dossierTemplate.getId())
+                                                             .dossierDictionaryOnly(false)
+                                                             .build());
+        assertThat(returnedtype1.getRank()).isEqualTo(100);
+
+        Assertions.assertThrows(FeignException.Conflict.class,
+                                () -> dictionaryClient.addType(CreateTypeValue.builder()
+                                                                       .type("test2")
+                                                                       .label("test2")
+                                                                       .hexColor("#fcba03")
+                                                                       .rank(100)
+                                                                       .hint(false)
+                                                                       .caseInsensitive(false)
+                                                                       .recommendation(false)
+                                                                       .addToDictionaryAction(true)
+                                                                       .dossierTemplateId(dossierTemplate.getId())
+                                                                       .dossierDictionaryOnly(false)
+                                                                       .build()));
+
+        var returnedtype2 = dictionaryClient.addType(CreateTypeValue.builder()
+                                                             .type("test2")
+                                                             .label("test2")
+                                                             .hexColor("#fcba13")
+                                                             .rank(50)
+                                                             .hint(false)
+                                                             .addToDictionaryAction(true)
+                                                             .dossierTemplateId(dossierTemplate.getId())
+                                                             .dossierDictionaryOnly(false)
+                                                             .build());
+        assertThat(returnedtype2.getRank()).isEqualTo(50);
+        Assertions.assertThrows(FeignException.Conflict.class, () -> {
+            var request = new UpdateTypeValue();
+            BeanUtils.copyProperties(returnedtype1, request);
+            request.setRank(50);
+            dictionaryClient.updateType(returnedtype1.getType(), returnedtype1.getDossierTemplateId(), request);
+        });
+
+    }
+
+
     @Test
     public void testAddEntriesToDossierTemplateDictionaryWithDossierDictionaryOnlyFlag() {
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java

@@ -0,0 +1,292 @@
+package com.iqser.red.service.peristence.v1.server.integration.tests;
+
+import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService;
+import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.WatermarkService;
+import com.iqser.red.service.persistence.management.v1.processor.service.export.DossierTemplateExportService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ExportDownloadRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.TypeRankSummary;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+//@DirtiesContext
+@ExtendWith(SpringExtension.class)
+public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
+
+    @MockBean
+    private DossierTemplateRepository dossierTemplateRepository;
+    @MockBean
+    private LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
+    @MockBean
+    private RulesPersistenceService rulesPersistenceService;
+
+    //    private DossierTemplatePersistenceService dossierTemplatePersistenceService;
+    @MockBean
+    private DossierAttributeConfigPersistenceService dossierAttributeConfigPersistenceService;
+    @MockBean
+    private DictionaryPersistenceService dictionaryPersistenceService;
+    @MockBean
+    private EntryPersistenceService entryPersistenceService;
+    @MockBean
+    private FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
+    @MockBean
+    private ReportTemplatePersistenceService reportTemplatePersistenceService;
+    @MockBean
+    private ColorsService colorsService;
+    @MockBean
+    private StorageService storageService;
+    @MockBean
+    private DossierStatusPersistenceService dossierStatusPersistenceService;
+    @MockBean
+    private WatermarkService watermarkService;
+    @MockBean
+    private FileManagementStorageService fileManagementStorageService;
+    @MockBean
+    private DownloadStatusPersistenceService downloadStatusPersistenceService;
+    @MockBean
+    private RabbitTemplate rabbitTemplate;
+    @MockBean
+    private TypeRepository typeRepository;
+
+    @MockBean
+    private DossierTemplateImportService dossierTemplateImportService;
+
+    private DossierTemplatePersistenceService dossierTemplatePersistenceService;
+    private DossierTemplateCloneService dossierTemplateCloneService;
+    private DossierTemplateExportService dossierTemplateExportService;
+    private RankDeDuplicationService rankDeDuplicationService;
+
+    private DossierTemplateManagementService dossierTemplateManagementService;
+    private String dossierTemplateId = "dossierTemplateId";
+    private String dossierId = "dossierId";
+    private String dossierTemplateName = "Clone of " + dossierTemplateId;
+
+    @Captor
+    private ArgumentCaptor<Integer> captor;
+
+
+    @BeforeEach
+    public void setupData() {
+
+        dossierTemplatePersistenceService = new DossierTemplatePersistenceService(dictionaryPersistenceService,
+                                                                                  dossierTemplateRepository,
+                                                                                  legalBasisMappingPersistenceService,
+                                                                                  rulesPersistenceService,
+                                                                                  typeRepository);
+        dossierTemplateCloneService = new DossierTemplateCloneService(dossierTemplateRepository,
+                                                                      legalBasisMappingPersistenceService,
+                                                                      rulesPersistenceService,
+                                                                      dossierTemplatePersistenceService,
+                                                                      dossierAttributeConfigPersistenceService,
+                                                                      dictionaryPersistenceService,
+                                                                      entryPersistenceService,
+                                                                      fileAttributeConfigPersistenceService,
+                                                                      reportTemplatePersistenceService,
+                                                                      colorsService,
+                                                                      storageService,
+                                                                      dossierStatusPersistenceService,
+                                                                      watermarkService,
+                                                                      fileManagementStorageService);
+        dossierTemplateExportService = new DossierTemplateExportService(dossierTemplatePersistenceService,
+                                                                        downloadStatusPersistenceService,
+                                                                        dossierAttributeConfigPersistenceService,
+                                                                        dossierStatusPersistenceService,
+                                                                        dictionaryPersistenceService,
+                                                                        fileAttributeConfigPersistenceService,
+                                                                        legalBasisMappingPersistenceService,
+                                                                        rulesPersistenceService,
+                                                                        fileManagementStorageService,
+                                                                        reportTemplatePersistenceService,
+                                                                        colorsService,
+                                                                        entryPersistenceService,
+                                                                        watermarkService,
+                                                                        rabbitTemplate);
+        dossierTemplateManagementService = new DossierTemplateManagementService(dossierTemplateExportService,
+                                                                                dossierTemplateImportService,
+                                                                                dossierTemplatePersistenceService,
+                                                                                dossierTemplateCloneService);
+        rankDeDuplicationService = new RankDeDuplicationService(dossierTemplateManagementService, dictionaryPersistenceService);
+
+        List<TypeRankSummary> typesValues = new ArrayList<>();
+        TypeRankSummary typeRank1 = new TypeRankSummary(dossierTemplateId, null, 50, 2);
+        TypeRankSummary typeRank2 = new TypeRankSummary(dossierTemplateId, null, 100, 1);
+        typesValues.add(typeRank1);
+        typesValues.add(typeRank2);
+        when(dictionaryPersistenceService.getTypeRankSummaryList(dossierTemplateId)).thenReturn(typesValues);
+
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testDownloadWithDuplicateRanks() {
+
+        // test the export of dossier template
+        ExportDownloadRequest exportDownloadRequest = new ExportDownloadRequest("userId", dossierTemplateId);
+        Assertions.assertThrows(BadRequestException.class, () -> dossierTemplateManagementService.prepareExportDownload(exportDownloadRequest));
+
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testCloneDossierTemplateWithDuplicateRanks() {
+
+        CloneDossierTemplateRequest cdtr = CloneDossierTemplateRequest.builder().name(dossierTemplateName).cloningUserId("user").ocrByDefault(true).removeWatermark(false).build();
+        DossierTemplateEntity dossierTemplateEntity = new DossierTemplateEntity();
+        dossierTemplateEntity.setId(dossierTemplateId);
+        when(dossierTemplateRepository.existsByName(cdtr.getName())).thenReturn(false);
+        when(dossierTemplateRepository.findById(dossierTemplateId)).thenReturn(Optional.of(dossierTemplateEntity));
+        Assertions.assertThrows(BadRequestException.class, () -> dossierTemplateManagementService.cloneDossierTemplate(dossierTemplateId, cdtr));
+
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testDeduplicateDossierTemplate() {
+
+        List<TypeEntity> allDossierTemplateTypes = new ArrayList<>();
+        allDossierTemplateTypes.add(createTypeEntity("type00", dossierTemplateId, 0)); // no change
+        allDossierTemplateTypes.add(createTypeEntity("type01", dossierTemplateId, 0)); // new rank will be 1
+        allDossierTemplateTypes.add(createTypeEntity("type02", dossierTemplateId, 0)); // new rank will be 2
+        allDossierTemplateTypes.add(createTypeEntity("type22", dossierTemplateId, 2)); // new rank will be 3
+        allDossierTemplateTypes.add(createTypeEntity("type0", dossierTemplateId, 49)); // no change
+        allDossierTemplateTypes.add(createTypeEntity("type1", dossierTemplateId, 50)); // no change
+        allDossierTemplateTypes.add(createTypeEntity("type2", dossierTemplateId, 50)); // new rank will be 51
+        allDossierTemplateTypes.add(createTypeEntity("type3", dossierTemplateId, 50)); // new rank will be 52
+        allDossierTemplateTypes.add(createTypeEntity("type4", dossierTemplateId, 50)); // new rank will be 53
+        allDossierTemplateTypes.add(createTypeEntity("type5", dossierTemplateId, 50)); // new rank will be 54
+        allDossierTemplateTypes.add(createTypeEntity("type6", dossierTemplateId, 51)); // new rank will be 55
+        allDossierTemplateTypes.add(createTypeEntity("type7", dossierTemplateId, 51)); // new rank will be 55
+        allDossierTemplateTypes.add(createTypeEntity("type8", dossierTemplateId, 51)); // new rank will be 57
+        allDossierTemplateTypes.add(createTypeEntity("type9", dossierTemplateId, 52)); // new rank will be 58
+        allDossierTemplateTypes.add(createTypeEntity("type10", dossierTemplateId, 52)); // new rank will be 59
+        allDossierTemplateTypes.add(createTypeEntity("type11", dossierTemplateId, 55)); // new rank will be 60
+        allDossierTemplateTypes.add(createTypeEntity("type12", dossierTemplateId, 100)); // no change
+
+        List<DossierTemplateEntity> dossierTemplates = new ArrayList<>();
+        DossierTemplateEntity dt = new DossierTemplateEntity();
+        dt.setId(dossierTemplateId);
+        dossierTemplates.add(dt);
+        when(dossierTemplateRepository.findAllWhereDeletedIsFalse()).thenReturn(dossierTemplates);
+        when(legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId)).thenReturn(Collections.emptyList());
+        when(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false)).thenReturn(allDossierTemplateTypes);
+        when(dictionaryPersistenceService.getAllDossierTypesForDossierTemplateAndType(dossierTemplateId, "type02", false)).thenReturn(List.of(createTypeEntity("type02",
+                                                                                                                                                               dossierTemplateId,
+                                                                                                                                                               dossierId,
+                                                                                                                                                               0)));
+
+        rankDeDuplicationService.deduplicate();
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type01", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 1);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type02", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 2);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type02", dossierTemplateId, dossierId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 2);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type22", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 3);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type2", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 51);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type3", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 52);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type4", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 53);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type5", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 54);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type6", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 55);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type7", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 56);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type8", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 57);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type9", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 58);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type10", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 59);
+
+        verify(dictionaryPersistenceService, times(1)).updateRankForType(eq(toTypeId("type11", dossierTemplateId)), captor.capture());
+        Assertions.assertEquals(captor.getValue(), 60);
+
+    }
+
+
+    private TypeEntity createTypeEntity(String type, String dossierTemplateId, int rank) {
+
+        return createTypeEntity(type, dossierTemplateId, null, rank);
+    }
+
+
+    private TypeEntity createTypeEntity(String type, String dossierTemplateId, String dossierId, int rank) {
+
+        TypeEntity typeEntity = new TypeEntity();
+        typeEntity.setId(toTypeId(type, dossierTemplateId, dossierId));
+        typeEntity.setType(type);
+        typeEntity.setDossierTemplateId(dossierTemplateId);
+        typeEntity.setRank(rank);
+        typeEntity.setDescription("description");
+        typeEntity.setLabel(type);
+        typeEntity.setHexColor("#ddddd");
+        typeEntity.setRecommendationHexColor("#cccccc");
+        typeEntity.setSkippedHexColor("#cccccc");
+        typeEntity.setAddToDictionaryAction(true);
+
+        return typeEntity;
+    }
+
+}

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateTest.java

@@ -1,42 +1,18 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
+import static org.assertj.core.api.Assertions.assertThat;
-import com.google.common.collect.Sets;
+import static org.junit.jupiter.api.Assertions.assertThrows;
-import com.iqser.red.service.peristence.v1.server.integration.client.*;
+import static org.junit.jupiter.api.Assertions.assertTrue;
-import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
-import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
-import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
-import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService;
-import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
-import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
-import com.iqser.red.service.persistence.management.v1.processor.service.export.ExportDownloadMessageReceiver;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.*;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-import feign.FeignException;
-import lombok.SneakyThrows;
-import org.jetbrains.annotations.NotNull;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.mock.web.MockMultipartFile;
 
-import java.io.*;
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -51,9 +27,56 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.assertj.core.api.Assertions.assertThat;
+import org.jetbrains.annotations.NotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
+import org.junit.jupiter.api.Disabled;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.mock.web.MockMultipartFile;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Sets;
+import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.DossierAttributeConfigClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.DossierStatusClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.LegalBasisClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.ReportTemplateClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.WatermarkClient;
+import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
+import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
+import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.export.ExportDownloadMessageReceiver;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierAttributesConfig;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStatusRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import feign.FeignException;
+import lombok.SneakyThrows;
 
 @SuppressWarnings("PMD")
 public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
@@ -187,6 +210,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
     public void testCloneDossierTemplate() {
 
         var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
 
         var type = CreateTypeValue.builder()
                 .type("t")
@@ -215,10 +239,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
                 .recommendation(false)
                 .caseInsensitive(false)
                 .description("t2.getDescription()")
-                .addToDictionaryAction(true)
+                .addToDictionaryAction(true).label("t2.getLabel()").hasDictionary(true).build();
-                .label("t2.getLabel()")
-                .hasDictionary(true)
-                .build();
 
         var createdType1 = dictionaryClient.addType(type);
         var createdType2 = dictionaryClient.addType(type2);
@@ -226,15 +247,31 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
         var loadedType1 = dictionaryClient.getDictionaryForType(createdType1.getType(), createdType1.getDossierTemplateId(), null);
         var loadedType2 = dictionaryClient.getDictionaryForType(createdType2.getType(), createdType2.getDossierTemplateId(), null);
 
+        // force types on dossier level
+        dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getId());
+        dictionaryClient.getDictionaryForType(type2.getType(), type2.getDossierTemplateId(), dossier.getId());
+
+        var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes();
+        assertThat(allTypes.size()).isEqualTo(4);
+        var typesWithRankOfType1 = allTypes.stream()
+                .filter(t -> t.getRank() == type.getRank())
+                .collect(Collectors.toList());
+        assertThat(typesWithRankOfType1.size()).isEqualTo(2);
+
+        var typesWithRankOfType2 = allTypes.stream()
+                .filter(t -> t.getRank() == type2.getRank())
+                .collect(Collectors.toList());
+        assertThat(typesWithRankOfType2.size()).isEqualTo(2);
+
         dictionaryClient.addEntry(createdType1.getType(), createdType1.getDossierTemplateId(), List.of("entry1", "entry2"), false, null, DictionaryEntryType.ENTRY);
         dictionaryClient.addEntry(createdType1.getType(), createdType1.getDossierTemplateId(), List.of("entry3", "entry4"), false, null, DictionaryEntryType.FALSE_POSITIVE);
         dictionaryClient.addEntry(createdType1.getType(), createdType1.getDossierTemplateId(), List.of("entry5", "entry6"), false, null, DictionaryEntryType.FALSE_RECOMMENDATION);
 
         dossierAttributeConfigClient.setDossierAttributesConfig(dossierTemplate.getId(),
-                new DossierAttributesConfig(List.of(DossierAttributeConfig.builder()
+                                                                new DossierAttributesConfig(List.of(DossierAttributeConfig.builder()
-                        .dossierTemplateId(dossierTemplate.getId())
+                                                                                                            .dossierTemplateId(dossierTemplate.getId())
-                        .editable(false)
+                                                                                                            .editable(false)
-                        .id("dossierAttributeId")
+                                                                                                            .id("dossierAttributeId")
                         .label("labelDossierAttribute")
                         .type(DossierAttributeType.TEXT)
                         .placeholder("placeholderDossier")

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java

@@ -6,7 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.when;
 
 import java.time.OffsetDateTime;
@@ -35,11 +34,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.EntityL
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
+import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.PendingDictionaryEntryFactory;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
@@ -91,10 +92,15 @@ public class EntityLogMergeTest {
 
     private EntityLogMergeService entityLogMergeService;
 
+
     @BeforeEach
     public void setUp() {
 
-        entityLogMergeService = new EntityLogMergeService(dictionaryPersistenceService, rabbitTemplate, fileStatusService, fileStatusPersistenceService);
+        entityLogMergeService = new EntityLogMergeService(dictionaryPersistenceService,
+                                                          rabbitTemplate,
+                                                          fileStatusService,
+                                                          fileStatusPersistenceService,
+                                                          new PendingDictionaryEntryFactory());
     }
 
 
@@ -114,30 +120,25 @@ public class EntityLogMergeTest {
 
         ManualRedactions manualRedactions = provideManualRedactions(entryToAddId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId);
 
-        var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId);
+        var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, false);
 
-        when(manualRedactionProviderService.getManualRedactions(fileId, true)).thenReturn(manualRedactions);
+        when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
-        when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder()
+        when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
-                .excluded(false)
-                .dossierStatusId(dossierTemplateId)
-                .id(fileId)
-                .build());
         when(fileManagementStorageService.getEntityLog(dossierId, fileId)).thenReturn(entityLog);
-        when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder()
+        when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
-                        .dossierTemplateId(dossierTemplateId)
-                .build());
         when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
         when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
         when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
 
-        EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder()
+        EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
-                .dossierTemplateId(dossierTemplateId)
-                .build());
 
         assertNotNull(response);
         assertFalse(response.getEntityLogEntry().isEmpty());
 
-        var optionalEntityLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToAddId)).findFirst();
+        var optionalEntityLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToAddId))
+                .findFirst();
         assertTrue(optionalEntityLogEntry.isPresent());
         var entityLogEntry = optionalEntityLogEntry.get();
         assertEquals(entityLogEntry.getType(), "manual");
@@ -145,48 +146,82 @@ public class EntityLogMergeTest {
         assertEquals(entityLogEntry.getState(), EntryState.APPLIED);
         assertEquals(entityLogEntry.getValue(), "Test");
         assertEquals(entityLogEntry.getReason(), "Reason");
-        assertEquals(entityLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.ADD_LOCALLY);
+        assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
+        assertEquals(entityLogEntry.getManualChanges()
+                             .get(0).getManualRedactionType(), ManualRedactionType.ADD);
 
-        var optionalRemoveEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRemoveId)).findFirst();
+        var optionalRemoveEntryLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRemoveId))
+                .findFirst();
         assertTrue(optionalRemoveEntryLogEntry.isPresent());
         var removeEntryLogEntry = optionalRemoveEntryLogEntry.get();
         assertEquals(removeEntryLogEntry.getEntryType(), EntryType.ENTITY);
         assertEquals(removeEntryLogEntry.getState(), EntryState.IGNORED);
-        assertEquals(removeEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.REMOVE_LOCALLY);
+        assertEquals(removeEntryLogEntry.getManualChanges()
-        assertEquals(removeEntryLogEntry.getChanges().get(0).getType(), ChangeType.REMOVED);
+                             .get(0).getManualRedactionType(), ManualRedactionType.REMOVE);
+        assertEquals(removeEntryLogEntry.getChanges()
+                             .get(0).getType(), ChangeType.REMOVED);
+        assertTrue(removeEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
-        var optionalResizeEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId)).findFirst();
+        var optionalResizeEntryLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId))
+                .findFirst();
         assertTrue(optionalResizeEntryLogEntry.isPresent());
         var resizeEntryLogEntry = optionalResizeEntryLogEntry.get();
-        assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[0], 2);
+        assertEquals(resizeEntryLogEntry.getPositions()
-        assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[1], 2);
+                             .get(0).getRectangle()[0], 2);
-        assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[2], 2);
+        assertEquals(resizeEntryLogEntry.getPositions()
-        assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[3], 2);
+                             .get(0).getRectangle()[1], 2);
-        assertEquals(resizeEntryLogEntry.getPositions().get(0).getPageNumber(), 1);
+        assertEquals(resizeEntryLogEntry.getPositions()
+                             .get(0).getRectangle()[2], 2);
+        assertEquals(resizeEntryLogEntry.getPositions()
+                             .get(0).getRectangle()[3], 2);
+        assertEquals(resizeEntryLogEntry.getPositions()
+                             .get(0).getPageNumber(), 1);
         assertEquals(resizeEntryLogEntry.getEntryType(), EntryType.ENTITY);
         assertEquals(resizeEntryLogEntry.getState(), EntryState.APPLIED);
-        assertEquals(resizeEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
+        assertEquals(resizeEntryLogEntry.getManualChanges()
-        assertEquals(resizeEntryLogEntry.getChanges().get(0).getType(), ChangeType.CHANGED);
+                             .get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
+        assertEquals(resizeEntryLogEntry.getChanges()
+                             .get(0).getType(), ChangeType.CHANGED);
+        assertTrue(resizeEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
-        var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId)).findFirst();
+        var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId))
+                .findFirst();
         assertTrue(optionalLegalBasisEntryLogEntry.isPresent());
         var legalBasisEntryLogEntry = optionalLegalBasisEntryLogEntry.get();
         assertEquals(legalBasisEntryLogEntry.getLegalBasis(), "New legal basis");
         assertEquals(legalBasisEntryLogEntry.getEntryType(), EntryType.ENTITY);
         assertEquals(legalBasisEntryLogEntry.getState(), EntryState.APPLIED);
-        assertEquals(legalBasisEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.LEGAL_BASIS_CHANGE);
+        assertEquals(legalBasisEntryLogEntry.getManualChanges()
-        assertEquals(legalBasisEntryLogEntry.getChanges().get(0).getType(), ChangeType.CHANGED);
+                             .get(0).getManualRedactionType(), ManualRedactionType.LEGAL_BASIS_CHANGE);
+        assertEquals(legalBasisEntryLogEntry.getChanges()
+                             .get(0).getType(), ChangeType.CHANGED);
+        assertTrue(legalBasisEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
-        var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId)).findFirst();
+        var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId))
+                .findFirst();
         assertTrue(optionalForceRedactionEntryLogEntry.isPresent());
         var forceRedactionEntryLogEntry = optionalForceRedactionEntryLogEntry.get();
         assertEquals(forceRedactionEntryLogEntry.getLegalBasis(), "Force");
         assertEquals(forceRedactionEntryLogEntry.getEntryType(), EntryType.ENTITY);
         assertEquals(forceRedactionEntryLogEntry.getState(), EntryState.APPLIED);
-        assertEquals(forceRedactionEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.FORCE_REDACT);
+        assertEquals(forceRedactionEntryLogEntry.getManualChanges()
-        assertEquals(forceRedactionEntryLogEntry.getChanges().get(0).getType(), ChangeType.CHANGED);
+                             .get(0).getManualRedactionType(), ManualRedactionType.FORCE);
+        assertEquals(forceRedactionEntryLogEntry.getChanges()
+                             .get(0).getType(), ChangeType.CHANGED);
+        assertTrue(forceRedactionEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
-        var optionalRectangleEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(rectangleToAddId)).findFirst();
+        var optionalRectangleEntryLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(rectangleToAddId))
+                .findFirst();
         assertTrue(optionalRectangleEntryLogEntry.isPresent());
         var rectangleEntryLogEntry = optionalRectangleEntryLogEntry.get();
         assertEquals(rectangleEntryLogEntry.getType(), "manual");
@@ -194,123 +229,238 @@ public class EntityLogMergeTest {
         assertEquals(rectangleEntryLogEntry.getState(), EntryState.APPLIED);
         assertEquals(rectangleEntryLogEntry.getValue(), "Test2");
         assertEquals(rectangleEntryLogEntry.getReason(), "Rectangle");
-        assertEquals(rectangleEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.ADD_LOCALLY);
+        assertEquals(rectangleEntryLogEntry.getManualChanges()
+                             .get(0).getManualRedactionType(), ManualRedactionType.ADD);
+        assertTrue(rectangleEntryLogEntry.getEngines().contains(Engine.MANUAL));
 
     }
 
-    private EntityLog provideEntityLog(String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId) {
+
+    @Test
+    public void testMergeEntityLogWithManualChangesOnDictRedaction() {
+
+        String fileId = "fileId";
+        String dossierId = "dossierId";
+        String dossierTemplateId = "dossierTemplateId";
+
+        String rectangleToAddId = UUID.randomUUID().toString();
+        String entryToRemoveId = UUID.randomUUID().toString();
+        String entryToResizeId = UUID.randomUUID().toString();
+        String entryLegalBasisId = UUID.randomUUID().toString();
+        String forceRedactionId = UUID.randomUUID().toString();
+
+        ManualRedactions manualRedactions = provideManualRedactions(entryLegalBasisId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId);
+
+        var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, true);
+
+        when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
+        when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
+        when(fileManagementStorageService.getEntityLog(dossierId, fileId)).thenReturn(entityLog);
+        when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+        when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
+        when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
+        when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
+
+        EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+
+        assertNotNull(response);
+        assertFalse(response.getEntityLogEntry().isEmpty());
+
+        var optionalResizeEntryLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId))
+                .findFirst();
+        assertTrue(optionalResizeEntryLogEntry.isPresent());
+        assertEquals(EntryType.ENTITY, optionalResizeEntryLogEntry.get().getEntryType());
+
+        var legalBasisEntries = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId))
+                .toList();
+        assertEquals(2, legalBasisEntries.size());
+        assertEquals(EntryState.REMOVED, legalBasisEntries.get(0).getState());
+        assertEquals(EntryState.APPLIED, legalBasisEntries.get(1).getState());
+
+        var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId))
+                .findFirst();
+        assertTrue(optionalForceRedactionEntryLogEntry.isPresent());
+        assertEquals(EntryState.APPLIED, optionalForceRedactionEntryLogEntry.get().getState());
+    }
+
+
+    @Test
+    public void testUnprocessedDictChangesAreDirectlyAfterOriginEntry() {
+
+        String fileId = "fileId";
+        String dossierId = "dossierId";
+        String dossierTemplateId = "dossierTemplateId";
+
+        String entryToRemoveId = UUID.randomUUID().toString();
+        String entryToResizeId = UUID.randomUUID().toString();
+        String entryLegalBasisId = UUID.randomUUID().toString();
+        String forceRedactionId = UUID.randomUUID().toString();
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+        List<Rectangle> positions = new ArrayList<>();
+        positions.add(new Rectangle(2, 2, 2, 2, 1));
+        manualRedactions.setResizeRedactions(Set.of(ManualResizeRedaction.builder()
+                                                            .fileId("fileId")
+                                                            .value("Random")
+                                                            .annotationId(entryToResizeId)
+                                                            .positions(positions)
+                                                            .requestDate(OffsetDateTime.now())
+                                                            .updateDictionary(true)
+                                                            .user("User")
+                                                            .fileId("file")
+                                                            .build()));
+
+        var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, false);
+
+        when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
+        when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
+        when(fileManagementStorageService.getEntityLog(dossierId, fileId)).thenReturn(entityLog);
+        when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+        when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
+        when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
+        when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
+
+        EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
+
+        var resizedEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId))
+                .findFirst()
+                .get();
+        int index = response.getEntityLogEntry().indexOf(resizedEntry);
+        assertEquals(response.getEntityLogEntry()
+                             .get(index + 1).getId(), resizedEntry.getId());
+        assertEquals(response.getEntityLogEntry()
+                             .get(index + 1).getState(), EntryState.PENDING);
+    }
+
+
+    private EntityLog provideEntityLog(String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId, boolean dictEntry) {
 
         List<Position> positions = new ArrayList<>();
         positions.add(new Position(1, 1, 1, 1, 1));
         return new EntityLog(1,
-                1,
+                             1,
-                Lists.newArrayList(EntityLogEntry.builder()
+                             Lists.newArrayList(EntityLogEntry.builder()
-                                .id(entryToRemoveId)
+                                                        .id(entryToRemoveId)
-                                .type("manual")
+                                                        .type("manual")
-                                .value("Luke Skywalker")
+                                                        .value("Luke Skywalker")
-                                .entryType(EntryType.ENTITY)
+                                                        .entryType(EntryType.ENTITY)
-                                .state(EntryState.APPLIED)
+                                                        .state(EntryState.APPLIED)
-                                .dictionaryEntry(true)
+                                                        .dictionaryEntry(dictEntry)
-                                .build(),
+                                                        .build(),
-                        EntityLogEntry.builder()
+                                                EntityLogEntry.builder()
-                                .id(entryToResizeId)
+                                                        .id(entryToResizeId)
-                                .type("manual")
+                                                        .type("manual")
-                                .value("Darth Vader")
+                                                        .value("Darth Vader")
-                                .entryType(EntryType.ENTITY)
+                                                        .entryType(EntryType.ENTITY)
-                                .state(EntryState.APPLIED)
+                                                        .state(EntryState.APPLIED)
-                                .dictionaryEntry(true)
+                                                        .dictionaryEntry(dictEntry)
-                                .positions(positions)
+                                                        .positions(positions)
-                                .build(),
+                                                        .build(),
-                        EntityLogEntry.builder()
+                                                EntityLogEntry.builder()
-                                .id(entryLegalBasisId)
+                                                        .id(entryLegalBasisId)
-                                .type("manual")
+                                                        .type("manual")
-                                .value("Darth Luke")
+                                                        .value("Darth Luke")
-                                .entryType(EntryType.ENTITY)
+                                                        .entryType(EntryType.ENTITY)
-                                .state(EntryState.APPLIED)
+                                                        .state(EntryState.APPLIED)
-                                .dictionaryEntry(true)
+                                                        .dictionaryEntry(dictEntry)
-                                .positions(positions)
+                                                        .positions(positions)
-                                .build(),
+                                                        .build(),
-                        EntityLogEntry.builder()
+                                                EntityLogEntry.builder()
-                                .id(forceRedactionId)
+                                                        .id(forceRedactionId)
-                                .type("manual")
+                                                        .type("manual")
-                                .value("Darth Luke")
+                                                        .value("Darth Luke")
-                                .entryType(EntryType.ENTITY)
+                                                        .entryType(EntryType.ENTITY)
-                                .state(EntryState.APPLIED)
+                                                        .state(EntryState.APPLIED)
-                                .dictionaryEntry(true)
+                                                        .dictionaryEntry(dictEntry)
-                                .positions(positions)
+                                                        .positions(positions)
-                                .build()),
+                                                        .build()),
-                null,
+                             Collections.emptyList(),
-                0,
+                             0,
-                0,
+                             0,
-                0,
+                             0,
-                0);
+                             0);
     }
 
-    private ManualRedactions provideManualRedactions(String entryToAddId, String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId, String fileId, String rectangleToAddId) {
+
+    private ManualRedactions provideManualRedactions(String entryToAddId,
+                                                     String entryToRemoveId,
+                                                     String entryToResizeId,
+                                                     String entryLegalBasisId,
+                                                     String forceRedactionId,
+                                                     String fileId,
+                                                     String rectangleToAddId) {
 
         List<Rectangle> positions = new ArrayList<>();
         positions.add(new Rectangle(2, 2, 2, 2, 1));
         return ManualRedactions.builder()
-                .entriesToAdd(Set.of(
+                .entriesToAdd(Set.of(ManualRedactionEntry.builder()
-                        ManualRedactionEntry.builder()
+                                             .positions(List.of(new Rectangle(1f, 2f, 3f, 4f, 1)))
-                                .positions(List.of(new Rectangle(1f, 2f, 3f, 4f, 1)))
+                                             .annotationId(entryToAddId)
-                                .annotationId(entryToAddId)
+                                             .value("Test")
-                                .value("Test")
+                                             .reason("Reason")
-                                .reason("Reason")
+                                             .addToDictionary(false)
-                                .addToDictionary(false)
+                                             .addToDossierDictionary(false)
-                                .addToDossierDictionary(false)
+                                             .fileId(fileId)
-                                .fileId(fileId)
+                                             .rectangle(false)
-                                .rectangle(false)
+                                             .fileId("file")
-                                .requestDate(OffsetDateTime.now())
+                                             .requestDate(OffsetDateTime.now())
-                                .dictionaryEntryType(DictionaryEntryType.ENTRY)
+                                             .dictionaryEntryType(DictionaryEntryType.ENTRY)
-                                .type("manual")
+                                             .type("manual")
-                        .build(),
+                                             .user("User")
-                        ManualRedactionEntry.builder()
+                                             .build(),
-                                .positions(List.of(new Rectangle(5f, 6f, 7f, 8f, 1)))
+                                     ManualRedactionEntry.builder()
-                                .annotationId(rectangleToAddId)
+                                             .positions(List.of(new Rectangle(5f, 6f, 7f, 8f, 1)))
-                                .value("Test2")
+                                             .annotationId(rectangleToAddId)
-                                .reason("Rectangle")
+                                             .value("Test2")
-                                .addToDictionary(false)
+                                             .reason("Rectangle")
-                                .addToDossierDictionary(false)
+                                             .addToDictionary(false)
-                                .fileId(fileId)
+                                             .addToDossierDictionary(false)
-                                .rectangle(true)
+                                             .fileId(fileId)
-                                .requestDate(OffsetDateTime.now())
+                                             .rectangle(true)
-                                .type("manual")
+                                             .requestDate(OffsetDateTime.now())
-                                .build()))
+                                             .type("manual")
-                .idsToRemove(Set.of(
+                                             .user("User")
-                        IdRemoval.builder()
+                                             .fileId("file")
-                                .annotationId(entryToRemoveId)
+                                             .build()))
-                                .requestDate(OffsetDateTime.now())
+                .idsToRemove(Set.of(IdRemoval.builder().annotationId(entryToRemoveId).requestDate(OffsetDateTime.now()).user("user").fileId("file").build()))
-                                .build()
+                .resizeRedactions(Set.of(ManualResizeRedaction.builder()
-                ))
+                                                 .fileId(fileId)
-                .resizeRedactions(Set.of(
+                                                 .value("Random")
-                        ManualResizeRedaction.builder()
+                                                 .annotationId(entryToResizeId)
-                                .fileId(fileId)
+                                                 .positions(positions)
-                                .value("Random")
+                                                 .requestDate(OffsetDateTime.now())
-                                .annotationId(entryToResizeId)
+                                                 .updateDictionary(false)
-                                .positions(positions)
+                                                 .addToAllDossiers(false)
-                                .requestDate(OffsetDateTime.now())
+                                                 .user("User")
-                                .updateDictionary(false)
+                                                 .fileId("file")
-                                .build()
+                                                 .build()))
-                ))
+                .legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
-                .legalBasisChanges(Set.of(
+                                                  .annotationId(entryLegalBasisId)
-                        ManualLegalBasisChange.builder()
+                                                  .value("Random")
-                                .annotationId(entryLegalBasisId)
+                                                  .legalBasis("New legal basis")
-                                .value("Random")
+                                                  .user("User")
-                                .legalBasis("New legal basis")
+                                                  .section("Section")
-                                .section("Section")
+                                                  .fileId("file")
-                                .requestDate(OffsetDateTime.now())
+                                                  .requestDate(OffsetDateTime.now())
-                                .build()
+                                                  .build()))
-                ))
+                .forceRedactions(Set.of(ManualForceRedaction.builder()
-                .forceRedactions(Set.of(
+                                                .annotationId(forceRedactionId)
-                        ManualForceRedaction.builder()
+                                                .fileId(fileId)
-                                .annotationId(forceRedactionId)
+                                                .legalBasis("Force")
-                                .fileId(fileId)
+                                                .requestDate(OffsetDateTime.now())
-                                .legalBasis("Force")
+                                                .user("User")
-                                .requestDate(OffsetDateTime.now())
+                                                .fileId("file")
-                                .build()
+                                                .build()))
-                ))
                 .build();
     }
+
 }
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java

@@ -379,6 +379,20 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
                                                             .value("value entry")
                                                             .state(EntryState.APPLIED)
                                                             .entryType(EntryType.ENTITY)
+                                                            .build(),
+                                                    EntityLogEntry.builder()
+                                                            .id("forceRedactionAnnotation")
+                                                            .type(type.getType())
+                                                            .value("value entry 2")
+                                                            .state(EntryState.APPLIED)
+                                                            .entryType(EntryType.ENTITY)
+                                                            .build(),
+                                                    EntityLogEntry.builder()
+                                                            .id("legalBasisChangeAnnotation")
+                                                            .type(type.getType())
+                                                            .value("value entry 3")
+                                                            .state(EntryState.APPLIED)
+                                                            .entryType(EntryType.ENTITY)
                                                             .build()),
                                             null,
                                             0,
@@ -416,7 +430,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
                                                                   .build()));
         manualRedactionClient.recategorizeBulk(dossierId,
                                                fileId,
-                                               Set.of(RecategorizationRequestModel.builder().annotationId(annotationId).comment("comment").type("new-type")
+                                               Set.of(RecategorizationRequestModel.builder()
+                                                              .annotationId(annotationId)
+                                                              .comment("comment")
+                                                              .type("new-type")
+                                                              .legalBasis("")
+                                                              .section("section")
                                                               .build()),
                                                false);
 
@@ -424,22 +443,22 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
 
         fileManagementClient.deleteFile(dossier.getId(), file.getId());
         var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
-        assertThat(softDeletedFiles.size()).isEqualTo(1);
+        assertThat(softDeletedFiles).hasSize(1);
         var activeFiles = fileClient.getDossierStatus(dossier.getId());
-        assertThat(activeFiles.size()).isEqualTo(0);
+        assertThat(activeFiles).isEmpty();
 
         fileManagementClient.restoreFiles(dossier.getId(), Sets.newHashSet(file.getId()));
         softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
-        assertThat(softDeletedFiles.size()).isEqualTo(0);
+        assertThat(softDeletedFiles).isEmpty();
         activeFiles = fileClient.getDossierStatus(dossier.getId());
-        assertThat(activeFiles.size()).isEqualTo(1);
+        assertThat(activeFiles).hasSize(1);
 
         fileManagementClient.hardDeleteFiles(dossier.getId(), List.of(file.getId()));
         softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
-        assertThat(softDeletedFiles.size()).isEqualTo(0);
+        assertThat(softDeletedFiles).isEmpty();
 
         activeFiles = fileClient.getDossierStatus(dossier.getId());
-        assertThat(activeFiles.size()).isEqualTo(0);
+        assertThat(activeFiles).isEmpty();
 
     }
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java

@@ -108,7 +108,7 @@ public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest
 
         var template = dossierTemplateTesterAndProvider.provideTestTemplate("test");
         var type1 = typeProvider.testAndProvideType(template, null, "t1");
-        var type2 = typeProvider.testAndProvideType(template, null, "t2");
+        var type2 = typeProvider.testAndProvideType(template, null, "t2", false, 50);
 
         List<DictionaryEntryEntity> type1Entries = entries.stream().map(s -> new DictionaryEntryEntity(0, s, 1, false, type1.getTypeId())).collect(Collectors.toList());
 

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/EntryState.java

@@ -4,5 +4,6 @@ public enum EntryState {
     APPLIED,
     SKIPPED,
     IGNORED,
-    REMOVED
+    REMOVED,
+    PENDING
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/ManualRedactionType.java

@@ -1,14 +1,24 @@
 package com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog;
 
 public enum ManualRedactionType {
+    @Deprecated
     ADD_LOCALLY,
-    ADD_TO_DICTIONARY,
+    @Deprecated
-    REMOVE_LOCALLY,
-    REMOVE_FROM_DICTIONARY,
     FORCE_REDACT,
+    @Deprecated
     FORCE_HINT,
+    @Deprecated
+    REMOVE_LOCALLY,
+
+    
+    ADD,
+    ADD_TO_DICTIONARY,
+    REMOVE,
+    REMOVE_FROM_DICTIONARY,
+    FORCE,
     RECATEGORIZE,
-    LEGAL_BASIS_CHANGE,
+    RECATEGORIZE_IN_DICTIONARY,
-    RESIZE, // Treat internally as a local resize. Documine already has documents with the value "RESIZE" in them, so we need it for backwards compatibility
+    @Deprecated LEGAL_BASIS_CHANGE,
+    RESIZE,
     RESIZE_IN_DICTIONARY
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java

@@ -4,6 +4,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -17,6 +19,8 @@ public class MigratedIds {
 
     List<Mapping> mappings;
 
+    List<ManualRedactionEntry> manualRedactionEntriesToAdd;
+
 
     public Map<String, String> buildOldToNewMapping() {
 

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/LegalBasisChangeRequest.java

@@ -5,6 +5,7 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+@Deprecated(forRemoval = true)
 @Data
 @Builder
 @AllArgsConstructor

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRedactions.java

@@ -1,11 +1,12 @@
 package com.iqser.red.service.persistence.service.v1.api.shared.model.annotations;
 
-import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
@@ -17,6 +18,7 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.NonNull;
 
 @Data
 @Builder
@@ -24,22 +26,60 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public class ManualRedactions {
 
+    @NonNull
     @Builder.Default
     private Set<IdRemoval> idsToRemove = new HashSet<>();
-
+    @NonNull
     @Builder.Default
     private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();
-
+    @NonNull
     @Builder.Default
     private Set<ManualForceRedaction> forceRedactions = new HashSet<>();
-
+    @NonNull
     @Builder.Default
     private Set<ManualRecategorization> recategorizations = new HashSet<>();
-
+    @NonNull
     @Builder.Default
     private Set<ManualLegalBasisChange> legalBasisChanges = new HashSet<>();
-
+    @NonNull
     @Builder.Default
     private Set<ManualResizeRedaction> resizeRedactions = new HashSet<>();
 
+
+    public List<BaseAnnotation> buildAll() {
+
+        List<BaseAnnotation> all = new ArrayList<>(idsToRemove.size()
+                                                   + entriesToAdd.size()
+                                                   + forceRedactions.size()
+                                                   + recategorizations.size()
+                                                   + legalBasisChanges.size()
+                                                   + resizeRedactions.size());
+        all.addAll(idsToRemove);
+        all.addAll(entriesToAdd);
+        all.addAll(forceRedactions);
+        all.addAll(recategorizations);
+        all.addAll(legalBasisChanges);
+        all.addAll(resizeRedactions);
+        return all;
+    }
+
+
+    public List<BaseAnnotation> buildAllSorted() {
+
+        return buildAll().stream()
+                .sorted(Comparator.comparing(BaseAnnotation::getRequestDate))
+                .toList();
+    }
+
+
+    public boolean isEmpty() {
+
+        return idsToRemove.isEmpty()
+               && entriesToAdd.isEmpty()
+               && forceRedactions.isEmpty()
+               && recategorizations.isEmpty()
+               && legalBasisChanges.isEmpty()
+               && resizeRedactions.isEmpty();
+    }
+
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RecategorizationRequest.java

@@ -28,6 +28,8 @@ public class RecategorizationRequest implements ManualRequestWithAddToDictionary
     boolean addToDictionary;
     boolean addToAllDossiers;
     private DictionaryEntryType dictionaryEntryType;
+    String legalBasis;
+    String section;
 
 
     @Override

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/BaseAnnotation.java

@@ -2,14 +2,13 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model.annotation
 
 import java.time.OffsetDateTime;
 
-import org.checkerframework.checker.units.qual.A;
-
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
+import lombok.NonNull;
 import lombok.experimental.SuperBuilder;
 
 @Data
@@ -17,15 +16,20 @@ import lombok.experimental.SuperBuilder;
 @NoArgsConstructor
 @AllArgsConstructor
 @EqualsAndHashCode
-public class BaseAnnotation {
+public abstract class BaseAnnotation {
 
+    @NonNull
     private String annotationId;
+    @NonNull
     private String fileId;
+    @NonNull
     private String user;
+    @NonNull
     private OffsetDateTime requestDate;
     private OffsetDateTime processedDate;
     private OffsetDateTime softDeletedTime;
 
+    public abstract boolean isLocal();
 
     @Deprecated(forRemoval = true)
     public boolean isApproved() {

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/IdRemoval.java

@@ -16,4 +16,11 @@ public class IdRemoval extends BaseAnnotation {
     private boolean removeFromDictionary;
     private boolean removeFromAllDossiers;
 
+
+    @Override
+    public boolean isLocal() {
+
+        return !(removeFromDictionary || removeFromAllDossiers);
+    }
+
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualForceRedaction.java

@@ -15,4 +15,11 @@ public class ManualForceRedaction extends BaseAnnotation {
 
     private String legalBasis;
 
+
+    @Override
+    public boolean isLocal() {
+
+        return true;
+    }
+
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualLegalBasisChange.java

@@ -6,6 +6,7 @@ import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.SuperBuilder;
 
+@Deprecated(forRemoval = true)
 @Data
 @SuperBuilder
 @NoArgsConstructor
@@ -17,4 +18,11 @@ public class ManualLegalBasisChange extends BaseAnnotation {
     private String value;
     private String legalBasis;
 
+
+    @Override
+    public boolean isLocal() {
+
+        return true;
+    }
+
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualRecategorization.java

@@ -14,5 +14,17 @@ import lombok.experimental.SuperBuilder;
 public class ManualRecategorization extends BaseAnnotation {
 
     private String type;
+    private String legalBasis;
+    private boolean addToDictionary;
+    private boolean addToAllDossiers;
+    private String section;
+    private String value;
+
+
+    @Override
+    public boolean isLocal() {
+
+        return !(addToDictionary || addToAllDossiers);
+    }
 
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualRedactionEntry.java

@@ -33,4 +33,11 @@ public class ManualRedactionEntry extends BaseAnnotation {
     private String sourceId;
     private DictionaryEntryType dictionaryEntryType;
 
+
+    @Override
+    public boolean isLocal() {
+
+        return !(addToDictionary || addToDossierDictionary);
+    }
+
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualResizeRedaction.java

@@ -25,4 +25,11 @@ public class ManualResizeRedaction extends BaseAnnotation {
     private Boolean updateDictionary;
     private boolean addToAllDossiers;
 
+
+    @Override
+    public boolean isLocal() {
+
+        return !(updateDictionary || addToAllDossiers);
+    }
+
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/type/DictionaryEntryType.java

@@ -1,7 +1,27 @@
 package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+
 public enum DictionaryEntryType {
     ENTRY,
-    FALSE_POSITIVE,
+    FALSE_POSITIVE {
-    FALSE_RECOMMENDATION
+        @Override
+        public EntryType toEntryType() {
+
+            return EntryType.FALSE_POSITIVE;
+        }
+    },
+    FALSE_RECOMMENDATION {
+        @Override
+        public EntryType toEntryType() {
+
+            return EntryType.FALSE_RECOMMENDATION;
+        }
+    };
+
+
+    public EntryType toEntryType() {
+
+        return EntryType.ENTITY;
+    }
 }

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/type/TypeRankSummary.java

@@ -0,0 +1,19 @@
+package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TypeRankSummary {
+
+    private String dossierTemplateId;
+    private String dossierId;
+    private int rank;
+    private long typesCount; // number of types with the same rank
+
+}

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/LegalBasisChangeRequestModel.java

@@ -6,6 +6,7 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.NonNull;
 
+@Deprecated(forRemoval = true)
 @Data
 @Builder
 @AllArgsConstructor

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RecategorizationRequestModel.java

@@ -19,5 +19,8 @@ public class RecategorizationRequestModel {
     String comment;
     boolean addToDictionary;
     boolean addToAllDossiers;
+    String legalBasis;
+    String section;
+    String value;
 
 }