Pull request #390: Port release to 340

Merge in RED/persistence-service from port-release-to-340 to master

* commit 'c17b7f75daf3bdbd5414ddb6e5e84e357171ee49':
  RED-3800 code cleanup
  Hotfix RED-3800 purge queue
  Hotfix RED-3800 reanslysis required status updates
  HOTFIX RED-3800 endpoint to reset text for all files that are not approved/deleted
  HOTFIX: RED-3800 ocr all files that have been ocrd
  HOTFIX: RED-3800 internal admin api
  migration manual run ctrl RED-3800
  NPE fix RED-3800
  RED-3800 Migration cleanup
  hotfix: migration to skip missing redaction logs (e.g. for deleted files w/o redaction log)

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java

@@ -19,10 +19,10 @@ public interface DossierTemplateStatsResource {
     String DOSSIER_TEMPLATE_ID = "dossierTemplateId";
     String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID + "}";
 
-    @PostMapping(value = REST_PATH+DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
+    @PostMapping(value = REST_PATH + DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
     List<DossierTemplateDictionaryStats> getDossierTemplateDictionaryStats(@RequestBody Set<String> dossierTemplateIds);
 
-    @PostMapping(value = REST_PATH+DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE )
+    @PostMapping(value = REST_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
     DossierTemplateStats getDossierTemplateStats(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId);
 
     @PostMapping(value = REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java

@@ -113,4 +113,7 @@ public class DossierEntity {
     @Column(updatable = false, insertable = false, name = "dossier_status_id")
     private String dossierStatusId;
 
+    public boolean isDeleted() {
+        return softDeletedTime != null || hardDeletedTime != null;
+    }
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java

@@ -29,20 +29,17 @@ public class MigrationPersistenceService {
 
         var migrations = migrationRepository.findByVersionGreaterThan(lastProcessedVersion);
         if (migrations == null || migrations.isEmpty()) {
-            if(version > lastProcessedVersion){
+            return version <= lastProcessedVersion;
-                return false;
-            }
-            return true;
         }
 
-        return migrations.stream().filter(m -> m.getVersion() == version).findFirst().isPresent();
+        return migrations.stream().anyMatch(m -> m.getVersion() == version);
     }
 
 
     public Long getLatestProcessedVersion() {
 
         var migrations = migrationRepository.findAll();
-        if (migrations == null || migrations.isEmpty()) {
+        if (migrations.isEmpty()) {
             return null;
         }
         return migrations.stream().sorted(Comparator.comparing(MigrationEntity::getVersion).reversed()).collect(Collectors.toList()).get(0).getVersion();

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java

@@ -1,8 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummaryResponse;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
@@ -20,7 +18,6 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
     List<TypeEntity> findByDossierId(String dossierId);
 
     @Query("select t from TypeEntity t where t.id = :typeId and t.softDeletedTime is null")
-
     Optional<TypeEntity> findByIdAndNotDeleted(String typeId);
 
     @Modifying

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java

@@ -1,25 +1,5 @@
 package com.iqser.red.service.peristence.v1.server.controller;
 
-import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
-import static java.util.stream.Collectors.toList;
-import static java.util.stream.Collectors.toSet;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.Predicate;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.transaction.Transactional;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
 import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities;
 import com.iqser.red.service.peristence.v1.server.service.StopwordService;
 import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator;
@@ -36,9 +16,25 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.transaction.Transactional;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
 
 @RestController
 @RequiredArgsConstructor
@@ -80,8 +76,8 @@ public class DictionaryController implements DictionaryResource {
         var currentVersion = getCurrentVersion(typeResult);
 
         if (removeCurrent) {
-            entryPersistenceService.deleteAllEntriesForTypeId(typeId,currentVersion+1,dictionaryEntryType);
+            entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType);
-            entryPersistenceService.addEntries(typeId,cleanEntries, currentVersion+1, dictionaryEntryType);
+            entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
         } else {
             entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
         }
@@ -136,9 +132,9 @@ public class DictionaryController implements DictionaryResource {
 
             var currentVersion = getCurrentVersion(typeResult);
 
-            entryPersistenceService.setVersion(typeId,currentVersion + 1, DictionaryEntryType.ENTRY);
+            entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
             entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
-            entryPersistenceService.setVersion(typeId,currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
+            entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
 
         }
 
@@ -160,7 +156,7 @@ public class DictionaryController implements DictionaryResource {
         checkForDuplicateLabels(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), typeRequest.getType(), typeRequest
                 .getLabel());
 
-        if (dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(),false)
+        if (dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), false)
                 .stream()
                 .anyMatch(typeResult -> typeRequest.getDossierId() != null && typeResult.getDossierId() != null && typeRequest
                         .getDossierId()
@@ -203,7 +199,7 @@ public class DictionaryController implements DictionaryResource {
             @PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
             @RequestParam(value = INCLUDE_DELETED_PARAMETER_NAME, required = false, defaultValue = "false") boolean includeDeleted) {
 
-        return convert(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId,includeDeleted), Type.class);
+        return convert(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, includeDeleted), Type.class);
     }
 
 
@@ -211,7 +207,7 @@ public class DictionaryController implements DictionaryResource {
     public List<Type> getAllTypesForDossier(@PathVariable(DOSSIER_ID_PARAMETER_NAME) String dossierId,
                                             @RequestParam(value = INCLUDE_DELETED_PARAMETER_NAME, required = false, defaultValue = "false") boolean includeDeleted) {
 
-        return convert(dictionaryPersistenceService.getAllTypesForDossier(dossierId,includeDeleted), Type.class);
+        return convert(dictionaryPersistenceService.getAllTypesForDossier(dossierId, includeDeleted), Type.class);
     }
 
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java

@@ -197,11 +197,11 @@ public class ReanalysisController implements ReanalysisResource {
 
         if (force) {
             filesToReanalyse.forEach(file -> {
-                fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true);
+                fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
             });
         } else {
             filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> {
-                fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true);
+                fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
             });
         }
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java

@@ -0,0 +1,151 @@
+package com.iqser.red.service.peristence.v1.server.internal;
+
+
+import com.iqser.red.service.peristence.v1.server.service.DossierService;
+import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
+import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
+import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.core.AmqpAdmin;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.stream.Collectors;
+
+@Slf4j
+@RestController
+@RequiredArgsConstructor
+@RequestMapping("/private/admin")
+public class AdminInterfaceController {
+
+    private final FileManagementStorageService fileManagementStorageService;
+    private final FileStatusService fileStatusService;
+    private final FileManagementServiceSettings fileManagementServiceSettings;
+    private final DossierService dossierService;
+    private final AmqpAdmin amqpAdmin;
+
+
+
+    @PostMapping("/reset-file")
+    public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
+
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
+
+        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
+
+
+    }
+
+
+    @PostMapping("/reset-image-info")
+    public void resetImageInfo(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
+
+        if (fileManagementServiceSettings.isImageServiceEnabled()) {
+            fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
+            fileStatusService.addToImageQueue(dossierId, fileId);
+        }
+    }
+
+
+    @PostMapping("/ocr")
+    public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
+
+        fileStatusService.updateLastOCRTime(fileId);
+        fileStatusService.setStatusOcrProcessing(dossierId, fileId);
+
+    }
+
+
+    @PostMapping("/ocr-all-files-that-have-already-been-ocrd")
+    public void forceOCR(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) {
+
+        if (dryRun) {
+            log.info("Dry run!");
+        }
+
+        var dossiers = dossierService.getAllDossiers();
+        for (var dossier : dossiers) {
+            if (!dossier.isDeleted()) {
+                var files = fileStatusService.getActiveFiles(dossier.getId());
+                var filesThatRequireOCR = files.stream()
+                        .filter(f -> !f.isExcluded())
+                        .filter(f -> !f.isExcludedFromAutomaticAnalysis())
+                        .filter(f -> !f.isSoftOrHardDeleted())
+                        .filter(f -> f.getLastOCRTime() != null)
+                        .filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList());
+
+
+                for (var file : filesThatRequireOCR) {
+                    log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}",
+                            file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime());
+
+                    if (!dryRun) {
+                        fileStatusService.updateLastOCRTime(file.getId());
+                        fileStatusService.setStatusOcrProcessing(file.getDossierId(), file.getId());
+                    }
+
+                }
+
+
+            }
+        }
+    }
+
+
+    @PostMapping("/reset-text-for-all-files")
+    public void resetText(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) {
+
+        if (dryRun) {
+            log.info("Dry run!");
+        }
+
+        var dossiers = dossierService.getAllDossiers();
+        for (var dossier : dossiers) {
+            if (!dossier.isDeleted()) {
+                var files = fileStatusService.getActiveFiles(dossier.getId());
+                var filesThatRequireTextReset = files.stream()
+                        .filter(f -> !f.isExcluded())
+                        .filter(f -> !f.isExcludedFromAutomaticAnalysis())
+                        .filter(f -> !f.isSoftOrHardDeleted())
+                        .filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList());
+
+
+                for (var file : filesThatRequireTextReset) {
+                    log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}",
+                            file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime());
+
+                    if (!dryRun) {
+
+                        var dossierId = file.getDossierId();
+                        var fileId = file.getId();
+
+                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID);
+                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
+                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
+                        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
+
+                        fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
+                    }
+
+                }
+
+
+            }
+        }
+    }
+
+    @PostMapping("/flush-queue")
+    public void resetText(@RequestParam() String queueName ) {
+        amqpAdmin.purgeQueue(queueName);
+    }
+
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.peristence.v1.server.migration;
 
-import org.springframework.beans.factory.annotation.Autowired;
-
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService;
-
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
 
 @Data
 @Slf4j
@@ -21,11 +19,14 @@ public abstract class Migration {
     @Setter
     private MigrationPersistenceService migrationPersistenceService;
 
-
     public void run() {
+        run(false);
+    }
+
+    public void run(boolean force) {
 
         var latestProcessedVersion = migrationPersistenceService.getLatestProcessedVersion();
-        if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion)) {
+        if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion) || force) {
             log.info("Starting migration with name {} and version {}", name, version);
             migrate();
             migrationPersistenceService.insertMigration(name, version);

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java

@@ -0,0 +1,28 @@
+package com.iqser.red.service.peristence.v1.server.migration;
+
+
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@Data
+@RestController
+@RequiredArgsConstructor
+@RequestMapping("/private/migration")
+public class MigrationController {
+
+    private final List<Migration> migrations;
+
+
+    @PostMapping
+    public void run(@RequestParam("migrationName") String migrationName, @RequestParam("force") boolean force) {
+        migrations.stream().filter(m -> m.getName().equalsIgnoreCase(migrationName) || m.getClass().getSimpleName().equalsIgnoreCase(migrationName))
+                .forEach(m -> m.run(force));
+    }
+
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DeleteRemovedManualAddRedactions7.java

@@ -1,20 +1,18 @@
 package com.iqser.red.service.peristence.v1.server.migration.migrations;
 
-import java.util.List;
-
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.peristence.v1.server.migration.Migration;
 import com.iqser.red.service.peristence.v1.server.service.ManualRedactionProviderService;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
-
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
 
 @Slf4j
 @Setter

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java

@@ -1,10 +1,5 @@
 package com.iqser.red.service.peristence.v1.server.migration.migrations;
 
-import java.util.ArrayList;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.peristence.v1.server.migration.Migration;
@@ -13,11 +8,13 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
 
 @Slf4j
 @Setter
@@ -56,8 +53,9 @@ public class DictionaryToEntityMigration2 extends Migration {
             files.forEach(file -> {
                 log.info("Start migration of file {}", file.getId());
                 if (file.getHardDeletedTime() == null) {
-                    var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
+
                     try {
+                        var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
                         var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId());
                         redactionLog.getRedactionLogEntry().forEach(entry -> {
 
@@ -69,20 +67,21 @@ public class DictionaryToEntityMigration2 extends Migration {
                             if (entry.getType().startsWith("recommendation_")) {
                                 entry.setType(entry.getType().substring(15));
                                 entry.setRecommendation(true);
-                                log.info("removed _recommendation");
+                                log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId());
                             }
 
                             newRedactionLogEntries.add(entry);
                         });
 
-                        redactionLog.setRedactionLogEntry(newRedactionLogEntries);
+                        if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) {
-                        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
+                            redactionLog.setRedactionLogEntry(newRedactionLogEntries);
-                        log.info("Stored dossierId: {} and fileId: {}", dossier.getId(), file.getId());
+                            fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
+                            log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
+                        }
                     } catch (JsonProcessingException e) {
                         throw new RuntimeException("Migration failed");
                     } catch (NotFoundException e) {
                         log.info("redactionLog {} does not exsist", file.getId());
-                        return;
                     }
                 }
             });

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/EntityTypesMigration4.java

@@ -1,20 +1,18 @@
 package com.iqser.red.service.peristence.v1.server.migration.migrations;
 
-import java.util.List;
-import java.util.Set;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.peristence.v1.server.migration.Migration;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
-
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Set;
 
 @Slf4j
 @Setter

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.peristence.v1.server.migration.migrations;
 
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.peristence.v1.server.migration.Migration;
 import com.iqser.red.service.peristence.v1.server.service.IndexingService;
-
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
 
 @Slf4j
 @Setter
@@ -16,6 +14,8 @@ public class IndexMigration1 extends Migration {
 
     private static final String NAME = "Recreate index and index all files";
     private static final long VERSION = 1;
+    @Autowired
+    private IndexingService indexingService;
 
 
     public IndexMigration1() {
@@ -23,11 +23,6 @@ public class IndexMigration1 extends Migration {
         super(NAME, VERSION);
     }
 
-
-    @Autowired
-    private IndexingService indexingService;
-
-
     @Override
     protected void migrate() {
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java

@@ -32,7 +32,6 @@ public class MigrateHighlights3 extends Migration {
     @Autowired
     private FileManagementStorageService fileManagementStorageService;
 
-
     public MigrateHighlights3() {
 
         super(NAME, VERSION);
@@ -48,24 +47,32 @@ public class MigrateHighlights3 extends Migration {
             files.forEach(file -> {
                 if (file.getHardDeletedTime() == null) {
 
-                    var untouchedExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.UNTOUCHED);
+                    try {
+                        if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) {
+                            var untouchedExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.UNTOUCHED);
 
-                    if (!untouchedExists) {
+                            if (!untouchedExists) {
-                        var originExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.ORIGIN);
+                                var originExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN);
 
-                        if (!originExists) {
+                                if (!originExists) {
-                            log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId());
+                                    log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId());
-                            return;
+                                    return;
+                                }
+
+                                fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED,
+                                        fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN));
+
+                            }
+                            var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder()
+                                    .fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build());
+
+                            fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights());
+                        } else {
+                            log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId());
                         }
-
+                    } catch (Exception e) {
-                        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED,
+                        log.warn("Failed to extract text highlights for document: {}", file.getId());
-                                fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN));
-
                     }
-                    var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder()
-                            .fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build());
-
-                    fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights());
                 }
             });
         });

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java

@@ -1,18 +1,16 @@
 package com.iqser.red.service.peristence.v1.server.migration.migrations;
 
-import java.util.ArrayList;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.peristence.v1.server.controller.ManualRedactionController;
 import com.iqser.red.service.peristence.v1.server.migration.Migration;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
-
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
 
 @Slf4j
 @Setter
@@ -59,6 +57,9 @@ public class RemoveFalsePositiveManualRedactions6 extends Migration {
                             }
                         });
                     }
+
+                    log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(), annotationIdsToRemove);
+
                     if (!annotationIdsToRemove.isEmpty()) {
                         annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id));
                     }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java

@@ -1,5 +1,17 @@
 package com.iqser.red.service.peristence.v1.server.migration.migrations;
 
+import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
+import com.iqser.red.service.peristence.v1.server.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Locale;
@@ -7,20 +19,6 @@ import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
-import com.iqser.red.service.peristence.v1.server.migration.Migration;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
-
-import lombok.Setter;
-import lombok.extern.slf4j.Slf4j;
-
 @Slf4j
 @Setter
 @Service
@@ -65,6 +63,7 @@ public class TypeToEntityMigration5 extends Migration {
                     .filter(t -> t.getType().equals("false_positive"))
                     .findFirst();
             if (falsePositive.isEmpty()) {
+                log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName());
                 return;
             }
             typeIdsToDelete.add(falsePositive.get().getId());
@@ -104,9 +103,7 @@ public class TypeToEntityMigration5 extends Migration {
                 log.info("Finished processing type {}", typeEntity.getType());
             });
 
-            typeIdsToDelete.forEach(typeIdToDelete -> {
+            typeIdsToDelete.forEach(typeIdToDelete -> dictionaryController.deleteType(typeIdToDelete));
-                dictionaryController.deleteType(typeIdToDelete);
-            });
             log.info("Finished false positive migration of dossierTemplate {}", dossierTemplate.getId());
         });
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java

@@ -1,12 +1,5 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.stereotype.Service;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@@ -16,10 +9,15 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.InputStream;
 
 @Slf4j
 @Service
@@ -86,7 +84,7 @@ public class FileManagementStorageService {
     }
 
 
-    public ImportedRedactions getImportedRedactions(String dossierId, String fileId){
+    public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
         InputStreamResource inputStreamResource;
         try {
             inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
@@ -101,11 +99,11 @@ public class FileManagementStorageService {
         }
     }
 
-    public boolean typeExists(String dossierId, String fileId, FileType type) {
+    public boolean objectExists(String dossierId, String fileId, FileType origin) {
-
+        return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, origin));
-        return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, type));
     }
 
+
     public void deleteObject(String dossierId, String fileId, FileType fileType) {
 
         storageService.deleteObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType));
@@ -117,4 +115,5 @@ public class FileManagementStorageService {
         storageService.deleteObject(storageId);
     }
 
+
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java

@@ -32,7 +32,7 @@ public class FileStatusProcessingUpdateService {
         switch (analyzeResult.getMessageType()) {
 
             case STRUCTURE_ANALYSE:
-                if (settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) {
+                if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
                     fileStatusService.addToNerQueue(dossierId, fileId);
                 } else {
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java

@@ -1,26 +1,10 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
-import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import javax.transaction.Transactional;
-
-import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Sets;
+import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
 import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
-import com.iqser.red.service.peristence.v1.server.controller.RulesController;
 import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
 import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
 import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
@@ -32,13 +16,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Inter
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
@@ -46,9 +25,20 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.MessageType;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import javax.transaction.Transactional;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
 
 @Slf4j
 @Service
@@ -59,7 +49,6 @@ public class FileStatusService {
     private final DossierPersistenceService dossierPersistenceService;
     private final RabbitTemplate rabbitTemplate;
     private final ObjectMapper objectMapper;
-    private final RulesController rulesController;
     private final ManualRedactionProviderService manualRedactionProviderService;
     private final FileManagementStorageService fileManagementStorageService;
     private final LegalBasisChangePersistenceService legalBasisChangePersistenceService;
@@ -129,7 +118,6 @@ public class FileStatusService {
     }
 
 
-
     public void setStatusProcessing(String fileId) {
 
         fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING);
@@ -199,9 +187,7 @@ public class FileStatusService {
             return;
         }
 
-        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS);
+        if (requiresStructureAnalysis) {
-
-        if(requiresStructureAnalysis) {
             log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
             fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
             fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
@@ -215,24 +201,28 @@ public class FileStatusService {
     protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse) {
 
         var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
-        var fileStatus = fileStatusPersistenceService.getStatus(fileId);
+        var fileEntity = fileStatusPersistenceService.getStatus(fileId);
 
-        if (fileStatus.isExcluded()) {
+        if (fileEntity.isExcluded()) {
-            log.debug("File {} is excluded", fileStatus.getId());
+            log.debug("File {} is excluded", fileEntity.getId());
             return;
         }
 
-        boolean reanalyse = isReanalyse(dossier, fileStatus);
+        var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
+        reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
 
-        if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId,FileType.IMAGE_INFO)) {
+
+        boolean reanalyse = fileModel.isReanalysisRequired();
+
+        if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
             log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
             addToImageQueue(dossierId, fileId);
             return;
         }
 
-        MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus());
+        MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus());
 
-        if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) {
+        if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
             log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
             addToNerQueue(dossierId, fileId);
             return;
@@ -241,26 +231,26 @@ public class FileStatusService {
         var analyseRequest = AnalyzeRequest.builder()
                 .messageType(messageType)
                 .dossierId(dossierId)
-                .analysisNumber(fileStatus.getNumberOfAnalyses() + 1)
+                .analysisNumber(fileModel.getNumberOfAnalyses() + 1)
                 .sectionsToReanalyse(sectionsToReanalyse)
                 .fileId(fileId)
                 .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
                 .dossierTemplateId(dossier.getDossierTemplateId())
-                .lastProcessed(fileStatus.getLastProcessed())
+                .lastProcessed(fileModel.getLastProcessed())
-                .fileAttributes(convertAttributes(fileStatus.getFileAttributes(), dossier.getDossierTemplateId()))
+                .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
-                .excludedPages(fileStatus.getExcludedPages())
+                .excludedPages(fileModel.getExcludedPages())
                 .build();
 
         log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);
 
-        if(messageType.equals(MessageType.REANALYSE)) {
+        if (messageType.equals(MessageType.REANALYSE)) {
             setStatusProcessing(fileId);
         } else {
             setStatusFullProcessing(fileId);
         }
 
         try {
-            if(priority){
+            if (priority) {
                 rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest));
             } else {
                 rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest));
@@ -271,21 +261,10 @@ public class FileStatusService {
     }
 
 
-    private boolean isReanalyse(DossierEntity dossier, FileEntity fileStatus) {
-
-        return !fileStatus.getProcessingStatus()
-                .equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
-                .equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus()
-                .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId())
-                && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
-                .isAfter(fileStatus.getLastFileAttributeChange()));
-    }
-
-
     private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
                                              ProcessingStatus processingStatus) {
 
-        if (!fileManagementStorageService.typeExists(dossierId, fileId, FileType.TEXT)) {
+        if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
             return MessageType.STRUCTURE_ANALYSE;
         }
         if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
@@ -416,7 +395,7 @@ public class FileStatusService {
     }
 
 
-    private void addToImageQueue(String dossierId, String fileId) {
+    public void addToImageQueue(String dossierId, String fileId) {
 
         setStatusImageAnalyzing(fileId);
         try {
@@ -454,7 +433,6 @@ public class FileStatusService {
     }
 
 
-
     public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) {
 
         fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights);

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java

@@ -127,23 +127,26 @@ public class ManualRedactionService {
             response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
         }
 
+
+        analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
+
         if (actionPerformed) {
             // in case of add to dict, there is no need to process surrounding text
             reprocess(dossierId, fileId);
+        }else {
+
+            var manualTextRedactions = convert(response.stream()
+                    .map(r -> getAddRedaction(fileId, r.getAnnotationId()))
+                    .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle())
+                    .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
+
+            if (!manualTextRedactions.isEmpty()) {
+                ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
+                addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
+            }
+
         }
 
-        var manualTextRedactions = convert(response.stream()
-                .map(r -> getAddRedaction(fileId, r.getAnnotationId()))
-                .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle())
-                .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
-
-        if (!manualTextRedactions.isEmpty()) {
-            ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
-            addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
-        }
-
-        analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
-
         return response;
     }
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java

@@ -33,16 +33,24 @@ public class ReanalysisRequiredStatusService {
 
 
     public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) {
-        return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel)).iterator().next();
+        return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), false).iterator().next();
+    }
+
+    public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel, boolean ignoreProcessingStates) {
+        return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), ignoreProcessingStates).iterator().next();
     }
 
     public List<FileModel> enhanceFileStatusWithAnalysisRequirements(List<FileModel> fileModels) {
+        return enhanceFileStatusWithAnalysisRequirements(fileModels, false);
+    }
+
+    public List<FileModel> enhanceFileStatusWithAnalysisRequirements(List<FileModel> fileModels, boolean ignoreProcessingStates) {
 
         Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>();
         Map<String, Long> dossierVersionMap = new HashMap<>();
         Map<String, DossierEntity> dossierMap = new HashMap<>();
         fileModels.forEach(entry -> {
-            var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
+            var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
             entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
             entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
         });
@@ -52,15 +60,18 @@ public class ReanalysisRequiredStatusService {
     }
 
     private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus,
+                                                           boolean ignoreProcessingStates,
                                                            Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
                                                            Map<String, Long> dossierVersionMap,
                                                            Map<String, DossierEntity> dossierMap) {
 
-        if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) {
+        if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus()) && !ignoreProcessingStates) {
             return new AnalysisRequiredResult(false, true);
         }
 
-        if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) {
+        if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus())
+                || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())
+                || ignoreProcessingStates) {
 
             switch (fileStatus.getWorkflowStatus()) {
                 case NEW:

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java

@@ -9,11 +9,9 @@ import lombok.extern.slf4j.Slf4j;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.springframework.amqp.core.AmqpAdmin;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
-import javax.annotation.PostConstruct;
 import java.util.List;
 
 @Slf4j
@@ -21,17 +19,9 @@ import java.util.List;
 @RequiredArgsConstructor
 public class AutomaticAnalysisJob implements Job {
 
-    @Value("${persistence-service.automaticAnalysis.pageFactor:500}")
-    private int pageFactor;
-    private final FileStatusService fileStatusService;
     private final AmqpAdmin amqpAdmin;
     private final FileManagementServiceSettings fileManagementServiceSettings;
-
+    private final FileStatusService fileStatusService;
-
-    @PostConstruct
-    protected void postConstruct() {
-        log.info("Automatic Analysis pageFactor: {}", pageFactor);
-    }
 
 
     @Override
@@ -55,12 +45,11 @@ public class AutomaticAnalysisJob implements Job {
                 var allStatusesIterator = allStatuses.iterator();
                 log.info("Files that require reanalysis: {}", allStatuses.size());
 
-                var worstCaseScenarioQueuedPages = 0;
+                var queuedFiles = 0;
 
-                while (worstCaseScenarioQueuedPages < pageFactor * consumerCount && allStatusesIterator.hasNext()) {
+                while (queuedFiles < (consumerCount * 5)) {
                     var next = allStatusesIterator.next();
                     // in case the file doesn't have numberOfPages set, we assume an average.
-                    worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages();
 
                     if (next.isFullAnalysisRequired()) {
                         log.info("Queued file: {} for automatic full analysis! ", next.getFilename());
@@ -69,6 +58,8 @@ public class AutomaticAnalysisJob implements Job {
                         log.info("Queued file: {} for automatic reanalysis! ", next.getFilename());
                         fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false);
                     }
+
+                    queuedFiles++;
                 }
             }
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java

@@ -1,38 +1,40 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
-import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
-import com.iqser.red.service.redaction.v1.model.ManualChange;
-import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
-import feign.FeignException;
-import org.assertj.core.util.Lists;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.springframework.beans.factory.annotation.Autowired;
-
 import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
+import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
 import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
 import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
+import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
+import com.iqser.red.service.redaction.v1.model.ManualChange;
+import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-
+import feign.FeignException;
 import lombok.SneakyThrows;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
 
 public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
@@ -57,6 +59,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
     @Autowired
     private FileManagementStorageService fileManagementStorageService;
 
+    @Autowired
+    private FileStatusPersistenceService fileStatusPersistenceService;
+
     @Autowired
     private ObjectMapper objectMapper;
 
@@ -142,6 +147,49 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
     }
 
+
+    @Test
+    @SneakyThrows
+    public void testAddToDictionaryRequiresReanalysis() {
+
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+
+        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
+
+        var file = fileTesterAndProvider.testAndProvideFile(dossier);
+
+        var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII");
+
+
+
+        // assume file is already proccessed once, test that add to dict triggers reanalysis
+        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.TEXT, "dummy".getBytes(StandardCharsets.UTF_8));
+        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "dummy".getBytes(StandardCharsets.UTF_8));
+        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "dummy".getBytes(StandardCharsets.UTF_8));
+        fileStatusPersistenceService.updateProcessingStatus(file.getId(),ProcessingStatus.PROCESSED);
+
+        var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder()
+                .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build()))
+                .section("section test")
+                .addToDictionary(true)
+                .addToDossierDictionary(false)
+                .status(AnnotationStatus.APPROVED)
+                .typeId(type.getId())
+                .user("user")
+                .reason("1")
+                .value("test")
+                .legalBasis("1")
+                .rectangle(true)
+                .textAfter("Text After")
+                .textBefore("Text Before")
+                .sourceId("SourceId")
+                .build())).iterator().next();
+
+
+        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
+        assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
+    }
+
     @Test
     @SneakyThrows
     public void testManualRedaction() {
@@ -295,7 +343,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                         .annotationStatus(AnnotationStatus.REQUESTED).build());
         var loadedRemoveRedaction2 = manualRedactionClient.getRemoveRedaction(file.getId(), removeRedaction2.getAnnotationId());
         assertThat(loadedRemoveRedaction2.getStatus()).isEqualTo(AnnotationStatus.REQUESTED);
-        assertThat(dictionaryClient.getDictionaryForType(type.getId(),null).getEntries().isEmpty());
+        assertThat(dictionaryClient.getDictionaryForType(type.getId(), null).getEntries().isEmpty());
         assertThat(loadedRemoveRedaction2.isRemoveFromDictionary()).isEqualTo(true);
 
         manualRedactionClient.updateRemoveRedactionStatus(dossier.getId(), file.getId(),
@@ -448,7 +496,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         assertThat(manualRedactions.getComments()).isNotEmpty();
         assertThat(manualRedactions.getResizeRedactions()).isNotEmpty();
 
-        List<String> annotationIds =  manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList());
+        List<String> annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList());
         manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), annotationIds);
 //        manualRedactions.getForceRedactions()
 //                .forEach(e -> manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), List.of(e.getAnnotationId())));