RED-4101: Perform preprocessing async via queue

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file;
 
 public enum ProcessingStatus {
-    ANALYSE, ERROR, FULLREPROCESS, IMAGE_ANALYZING, INDEXING, NER_ANALYZING, OCR_PROCESSING, PROCESSED, PROCESSING, REPROCESS, SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING
+    ANALYSE, ERROR, FULLREPROCESS, IMAGE_ANALYZING, INDEXING, NER_ANALYZING, OCR_PROCESSING, PROCESSED, PROCESSING, REPROCESS, SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING, PRE_PROCESSING, PRE_PROCESSED, PRE_PROCESSING_FAILED
 
 }

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.persistence.service.v1.api.resources;
 
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -8,6 +7,8 @@ import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
+import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 
 public interface FileStatusProcessingUpdateResource {
 
@@ -19,6 +20,17 @@ public interface FileStatusProcessingUpdateResource {
     String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
 
 
+    @ResponseStatus(value = HttpStatus.OK)
+    @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-successful", consumes = MediaType.APPLICATION_JSON_VALUE)
+    void preprocessingSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId,
+                                 @RequestBody UntouchedDocumentResponse untouchedDocumentResponse);
+
+
+    @ResponseStatus(value = HttpStatus.OK)
+    @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed")
+    void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
+
+
     @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-successful", consumes = MediaType.APPLICATION_JSON_VALUE)
     void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId,

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -35,7 +35,7 @@ public class FileStatusPersistenceService {
     private final DossierPersistenceService dossierService;
 
 
-    public void createStatus(String dossierId, String fileId, String filename, String uploader, boolean hasHighlights) {
+    public void createStatus(String dossierId, String fileId, String filename, String uploader) {
 
         OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
         FileEntity file = new FileEntity();
@@ -50,11 +50,30 @@ public class FileStatusPersistenceService {
         file.setLastUploaded(now);
         file.setLastUpdated(now);
         file.setFileManipulationDate(now);
-        file.setHasHighlights(hasHighlights);
         fileRepository.save(file);
     }
 
 
+    @Transactional
+    public void updateProcessingStatusPreprocessed(String fileId, boolean hasHighlights) {
+
+        if (isFileDeleted(fileId)) {
+            return;
+        }
+        fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
+    }
+
+
+    @Transactional
+    public void updateProcessingStatusPreprocessingFailed(String fileId) {
+
+        if (isFileDeleted(fileId)) {
+            return;
+        }
+        fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_FAILED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+    }
+
+
     @Transactional
     public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion,
                                        long legalBasisVersion, long duration, long dossierDictionaryVersion,
@@ -300,20 +319,20 @@ public class FileStatusPersistenceService {
 
 
     @Transactional
-    public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights, boolean keepManualRedactions) {
+    public void overwriteFile(String fileId, String uploader, String filename, boolean keepManualRedactions) {
 
         int countUpdate = 0;
 
         if(keepManualRedactions) {
-            countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, OffsetDateTime.now()
+            countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, OffsetDateTime.now()
-                    .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
+                    .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         } else {
 
             fileRepository.findById(fileId).ifPresent((file) -> {
                 file.setExcludedPages(new HashSet<>());
             });
 
-            countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
+            countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         }
 
         if (countUpdate == 0) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -56,6 +56,13 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
             " where f.id = :fileId")
     void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate);
 
+
+    @Modifying(clearAutomatically = true)
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.hasHighlights = :hasHighlights " +
+            "where f.id = :fileId")
+    void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights);
+
+
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated " +
             "where f.id = :fileId")
@@ -129,9 +136,9 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
             + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
             + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
             + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
-            + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights, f.excludedFromAutomaticAnalysis = false " + "where f.id = :fileId")
+            + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false " + "where f.id = :fileId")
     int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded,
-                      OffsetDateTime lastUpdated, boolean hasHighlights);
+                      OffsetDateTime lastUpdated);
 
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, "
@@ -140,9 +147,9 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
             + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
             + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
             + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
-            + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " + "where f.id = :fileId")
+            + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false " + "where f.id = :fileId")
     int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded,
-                      OffsetDateTime lastUpdated, boolean hasHighlights);
+                      OffsetDateTime lastUpdated);
 
     @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" +
             " and ((f.deleted is not null and f.hardDeletedTime is null) or " +

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java

@@ -41,6 +41,9 @@ public class MessagingConfiguration {
     public static final String NER_SERVICE_RESPONSE_QUEUE = "entity_response_queue";
     public static final String NER_SERVICE_DLQ = "entity_dead_letter_queue";
 
+    public static final String PRE_PROCESSING_QUEUE = "preprocessingQueue";
+    public static final String PRE_PROCESSING_DLQ = "preprocessingDLQ";
+
 
     @Bean
     public Queue nerRequestQueue() {
@@ -218,4 +221,22 @@ public class MessagingConfiguration {
         return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build();
     }
 
+    @Bean
+    public Queue preprocessingQueue() {
+
+        return QueueBuilder.durable(PRE_PROCESSING_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", PRE_PROCESSING_DLQ)
+                .withArgument("x-max-priority", 2)
+                .maxPriority(2)
+                .build();
+    }
+
+
+    @Bean
+    public Queue preprocessingDeadLetterQueue() {
+
+        return QueueBuilder.durable(PRE_PROCESSING_DLQ).build();
+    }
+
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java

@@ -4,6 +4,7 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
 import com.iqser.red.service.peristence.v1.server.service.FileStatusProcessingUpdateService;
 import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
@@ -19,6 +20,21 @@ public class FileStatusProcessingUpdateController implements FileStatusProcessin
     private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
 
 
+    public void preprocessingSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                                        @PathVariable(FILE_ID) String fileId,
+                                        @RequestBody UntouchedDocumentResponse untouchedDocumentResponse) {
+
+        fileStatusProcessingUpdateService.preprocessingSuccessful(dossierId, fileId, untouchedDocumentResponse);
+    }
+
+
+    public void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                                    @PathVariable(FILE_ID) String fileId) {
+
+        fileStatusProcessingUpdateService.preprocessingFailed(dossierId, fileId);
+    }
+
+
     public void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
                                    @PathVariable(FILE_ID) String fileId, @RequestBody AnalyzeResult analyzeResult) {
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java

@@ -69,27 +69,17 @@ public class FileService {
             }
         }
 
-        UntouchedDocumentResponse untouchedDocumentResponse;
         ApplicationConfigurationEntity applicationConfigurationEntity = applicationConfigService.getApplicationConfig();
-        try {
-            untouchedDocumentResponse  = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder()
-                    .dossierId(request.getDossierId())
-                    .fileId(request.getFileId()).fileName(request.getFilename())
-                    .removeDigitalSignaturesOnUpload(applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload()).build());
 
-        } catch (FeignException e) {
-            log.warn("Failed to optimize file: {}", request.getFilename(), e);
-            throw new BadRequestException(e.getMessage());
-        }
 
         if (existingStatus != null) {
             // the file is already uploaded, just reanalyse it.
 
-            fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights(), keepManualRedactions);
+            fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), keepManualRedactions, applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload());
         } else {
             // the file is new, should create a new status for it.
             log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());
-            fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights());
+            fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload());
         }
         return new JSONPrimitive<>(request.getFileId());
     }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
+import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
 import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
@@ -8,6 +9,8 @@ import com.iqser.red.service.search.v1.model.IndexMessageType;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.retry.support.RetryTemplate;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
 @Slf4j
@@ -69,6 +72,21 @@ public class FileStatusProcessingUpdateService {
     }
 
 
+    public void preprocessingSuccessful(String dossierId, String fileId,
+                                 UntouchedDocumentResponse untouchedDocumentResponse){
+
+        fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights());
+    }
+
+
+
+    public void preprocessingFailed(String dossierId, String fileId){
+
+        fileStatusService.updateProcessingStatusPreprocessingFailed(dossierId, fileId);
+    }
+
+
+
     public void ocrSuccessful(String dossierId, String fileId) {
 
         retryTemplate.execute(retryContext -> {

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java

@@ -4,14 +4,17 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Sets;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
+import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
 import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
 import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
 import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
 import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
+import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
@@ -24,14 +27,19 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.MessageType;
+
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
 import javax.transaction.Transactional;
+
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -61,9 +69,12 @@ public class FileStatusService {
     private final FileManagementServiceSettings settings;
     private final ReanalysisRequiredStatusService reanalysisRequiredStatusService;
     private final ViewedPagesPersistenceService viewedPagesPersistenceService;
+    private final ApplicationConfigService applicationConfigService;
+
 
     @Transactional
     public List<FileModel> getAllRelevantStatusesForReanalysisScheduler() {
+
         var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler();
         var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
         return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList)
@@ -72,6 +83,7 @@ public class FileStatusService {
                 .collect(Collectors.toList());
     }
 
+
     @Transactional
     public List<FileModel> getActiveFiles(String dossierId) {
 
@@ -89,7 +101,9 @@ public class FileStatusService {
         return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
     }
 
+
     public List<FileModel> getSoftDeletedForDossierList(List<String> dossierIds) {
+
         var fileEntities = fileStatusPersistenceService.getSoftDeletedFiles(dossierIds);
         var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
         return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
@@ -99,12 +113,31 @@ public class FileStatusService {
 
     @Transactional
     public FileModel getStatus(String fileId) {
+
         var fileEntity = fileStatusPersistenceService.getStatus(fileId);
         var converted = convert(fileEntity, FileModel.class, new FileModelMapper());
         return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(converted);
     }
 
 
+    public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights) {
+
+        fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights);
+        addToAnalysisQueue(dossierId, fileId, false, Set.of());
+    }
+
+
+    public void updateProcessingStatusPreprocessingFailed(String dossierId, String fileId) {
+
+        // TODO add better logic than always reprocess.
+
+        fileStatusPersistenceService.updateProcessingStatusPreprocessingFailed(fileId);
+        ApplicationConfigurationEntity applicationConfigurationEntity = applicationConfigService.getApplicationConfig();
+        var fileEntity = fileStatusPersistenceService.getStatus(fileId);
+        addToPreprocessingQueue(dossierId, fileId, fileEntity.getFilename(), applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload());
+    }
+
+
     public void setExcludedPages(String fileId, Set<Integer> excludedPages) {
 
         fileStatusPersistenceService.setExcludedPages(fileId, excludedPages);
@@ -117,6 +150,12 @@ public class FileStatusService {
     }
 
 
+    public void setStatusPreProcessing(String fileId) {
+
+        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING);
+    }
+
+
     public void setStatusProcessing(String fileId) {
 
         fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING);
@@ -177,7 +216,8 @@ public class FileStatusService {
 
 
     @Transactional
-    public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) {
+    public void setStatusFullReprocess(String dossierId, String fileId, boolean priority,
+                                       boolean requiresStructureAnalysis) {
 
         FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
 
@@ -196,8 +236,26 @@ public class FileStatusService {
     }
 
 
+    @SneakyThrows
+    public void addToPreprocessingQueue(String dossierId, String fileId, String filename,
+                                        boolean removeDigitalSignatures) {
+
+        var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder()
+                .dossierId(dossierId)
+                .fileId(fileId)
+                .fileName(filename)
+                .removeDigitalSignaturesOnUpload(removeDigitalSignatures)
+                .build();
+
+        setStatusPreProcessing(fileId);
+
+        rabbitTemplate.convertAndSend(MessagingConfiguration.PRE_PROCESSING_QUEUE, objectMapper.writeValueAsString(processUntouchedDocumentRequest));
+    }
+
+
     @Transactional
-    protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse) {
+    protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority,
+                                      Set<Integer> sectionsToReanalyse) {
 
         var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
         var fileEntity = fileStatusPersistenceService.getStatus(fileId);
@@ -210,7 +268,6 @@ public class FileStatusService {
         var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
         reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
 
-
         boolean reanalyse = fileModel.isReanalysisRequired();
 
         if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
@@ -270,7 +327,8 @@ public class FileStatusService {
             return MessageType.ANALYSE;
         }
         if (reanalyse) {
-            if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) {
+            if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed()
+                    .isBefore(fileModel.getLastFileAttributeChange())) {
                 return MessageType.ANALYSE;
             }
             return MessageType.REANALYSE;
@@ -294,10 +352,11 @@ public class FileStatusService {
 
 
     @Transactional
-    public void createStatus(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) {
+    public void createStatus(String dossierId, String fileId, String uploader, String filename,
+                             boolean removeDigitalSignatures) {
 
-        fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, hasHighlights);
+        fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader);
-        addToAnalysisQueue(dossierId, fileId, false, Set.of());
+        addToPreprocessingQueue(dossierId, fileId, filename, removeDigitalSignatures);
     }
 
 
@@ -435,7 +494,8 @@ public class FileStatusService {
     }
 
 
-    public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights, boolean keepManualRedactions) {
+    public void overwriteFile(String dossierId, String fileId, String uploader, String filename,
+                              boolean keepManualRedactions, boolean removeDigitalSignatures) {
 
         fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
         fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID);
@@ -443,16 +503,16 @@ public class FileStatusService {
         fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
         fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
 
-        if(keepManualRedactions) {
+        if (keepManualRedactions) {
-            fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights, true);
+            fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, true);
         } else {
-            fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights, false);
+            fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, false);
             deleteManualRedactions(dossierId, fileId);
         }
 
         viewedPagesPersistenceService.deleteForFile(fileId);
 
-        setStatusFullReprocess(dossierId, fileId, false, true);
+        addToPreprocessingQueue(dossierId, fileId, filename, removeDigitalSignatures);
     }
 
 
@@ -513,7 +573,8 @@ public class FileStatusService {
 
             var config = fileAttributeConfigs.stream()
                     .filter(fac -> fac.getId().equals(fileAttribute.getFileAttributeId().getFileAttributeConfigId()))
-                    .findFirst().orElseThrow(() -> new InternalServerErrorException("File Attribute config not defined for attribute :" + fileAttribute.getFileAttributeId()));
+                    .findFirst()
+                    .orElseThrow(() -> new InternalServerErrorException("File Attribute config not defined for attribute :" + fileAttribute.getFileAttributeId()));
 
             com.iqser.red.service.redaction.v1.model.FileAttribute attribute = com.iqser.red.service.redaction.v1.model.FileAttribute.builder()
                     .id(fileAttribute.getFileAttributeId().getFileAttributeConfigId())

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java

@@ -70,7 +70,7 @@ public class ReanalysisRequiredStatusService {
         }
 
         if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus())
-                || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())
+                || ProcessingStatus.PRE_PROCESSED.equals(fileStatus.getProcessingStatus())
                 || ignoreProcessingStates) {
 
             switch (fileStatus.getWorkflowStatus()) {

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java

@@ -60,6 +60,7 @@ public class FileTesterAndProvider {
                 List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()),
                 null, 0, 0, 0, 0)));
         fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, objectMapper.writeValueAsBytes(new SectionGrid()));
+        fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.ORIGIN, objectMapper.writeValueAsBytes("bytes of the file"));
 
         return file;
     }

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java

@@ -115,7 +115,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest {
         assertThat(dossierStats.isHasSuggestionsFilePresent()).isTrue();
         assertThat(dossierStats.isHasUpdatesFilePresent()).isTrue();
         assertThat(dossierStats.isHasNoFlagsFilePresent()).isFalse();
-        assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.FULL_PROCESSING)).isEqualTo(2);
+        assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.PRE_PROCESSING)).isEqualTo(2);
         assertThat(dossierStats.getLastFileUpdateDate()).isEqualTo(NOW);
         assertThat(dossierStats.getFileManipulationDate()).isEqualTo(OLDER_2);
     }