RED-4343: Added processingErrorCounter to FileEntity and limit retries for failure files to 5 (via setting)

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java

@@ -33,8 +33,7 @@ public interface FileStatusProcessingUpdateResource {
 
     @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-successful", consumes = MediaType.APPLICATION_JSON_VALUE)
-    void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId,
+    void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AnalyzeResult analyzeResult);
-                            @RequestBody AnalyzeResult analyzeResult);
 
 
     @ResponseStatus(value = HttpStatus.OK)

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java

@@ -1,13 +1,18 @@
 package com.iqser.red.service.persistence.service.v1.api.resources;
 
+import java.util.Set;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
 import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.DeleteImportedRedactionsRequest;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.Set;
 
 @ResponseStatus(value = HttpStatus.NO_CONTENT)
 public interface ReanalysisResource {
@@ -55,9 +60,11 @@ public interface ReanalysisResource {
     @PostMapping(value = IMPORT_REDACTIONS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
     void importRedactions(@RequestBody ByteContentDocument documentRequest);
 
+
     @PostMapping(value = IMPORT_REDACTIONS_PATH + "/delete", consumes = MediaType.APPLICATION_JSON_VALUE)
     void deleteImportedRedactions(@RequestBody DeleteImportedRedactionsRequest deleteImportedRedactionsRequest);
 
+
     @PostMapping(value = CONVERT_TEXT_HIGHLIGHTS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
     void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest);
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java

@@ -169,6 +169,9 @@ public class FileEntity {
     @Column(name = "dossier_id")
     private String dossierId;
 
+    @Column
+    private Integer processingErrorCounter;
+
     public boolean isSoftDeleted() {
         return deleted != null;
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -52,6 +52,7 @@ public class FileStatusPersistenceService {
         file.setLastUploaded(now);
         file.setLastUpdated(now);
         file.setFileManipulationDate(now);
+        file.setProcessingErrorCounter(0);
         fileRepository.save(file);
     }
 
@@ -62,8 +63,8 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED,
+        fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, OffsetDateTime.now()
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights, fileSize);
+                .truncatedTo(ChronoUnit.MILLIS), hasHighlights, fileSize, calculateProcessingErrorCounter(fileId, ProcessingStatus.PRE_PROCESSED));
     }
 
 
@@ -73,27 +74,26 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_FAILED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+        fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_FAILED, OffsetDateTime.now()
+                .truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PRE_PROCESSING_FAILED));
     }
 
 
     @Transactional
-    public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion,
+    public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion, long legalBasisVersion, long duration,
-                                       long legalBasisVersion, long duration, long dossierDictionaryVersion,
+                                       long dossierDictionaryVersion, int analysisVersion, int analysisNumber) {
-                                       int analysisVersion, int analysisNumber) {
 
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion,
+        fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now()
-                legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now()
+                .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
-                        .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
+                .truncatedTo(ChronoUnit.MILLIS), analysisNumber, calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
     }
 
 
     @Transactional
-    public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages,
+    public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates) {
-                            boolean hasSuggestions, boolean hasComments, boolean hasUpdates) {
 
         if (isFileDeleted(fileId)) {
             return;
@@ -102,12 +102,6 @@ public class FileStatusPersistenceService {
     }
 
 
-    private boolean isFileDeleted(String fileId) {
-
-        return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
-    }
-
-
     @Transactional
     public void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, boolean approval) {
 
@@ -115,12 +109,10 @@ public class FileStatusPersistenceService {
             return;
         }
         if (approval) {
-            fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now()
+            fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
-                    .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
                     .truncatedTo(ChronoUnit.MILLIS), true);
         } else {
-            fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now()
+            fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), null);
-                    .truncatedTo(ChronoUnit.MILLIS), null);
         }
 
     }
@@ -134,11 +126,11 @@ public class FileStatusPersistenceService {
         }
         if (processingStatus == ProcessingStatus.PROCESSED) {
             // In case the file is updated to "processed", "lastProcessed" date should be updated to "now"
-            fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
+            fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
-                    .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+                    .truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
         } else {
             fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
-                    .truncatedTo(ChronoUnit.MILLIS));
+                    .truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, processingStatus));
         }
     }
 
@@ -170,8 +162,8 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
+        fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now()
-                .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+                .truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
     }
 
 
@@ -236,12 +228,10 @@ public class FileStatusPersistenceService {
 
     public List<FileEntity> getActiveFiles(String dossierId) {
 
-        return fileRepository.findByDossierId(dossierId)
+        return fileRepository.findByDossierId(dossierId).stream().filter(f -> !f.isSoftOrHardDeleted()).collect(Collectors.toList());
-                .stream()
-                .filter(f -> !f.isSoftOrHardDeleted())
-                .collect(Collectors.toList());
     }
 
+
     public List<FileEntity> getSoftDeletedFiles(List<String> dossierIds) {
 
         return fileRepository.getSoftDeletedFiles(dossierIds);
@@ -264,8 +254,7 @@ public class FileStatusPersistenceService {
     @Transactional
     public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
 
-        int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
+        int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
-                .truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -282,8 +271,7 @@ public class FileStatusPersistenceService {
         }, () -> {
             throw new NotFoundException("Unknown file=" + fileId);
         });
-        fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now()
+        fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), null);
-                .truncatedTo(ChronoUnit.MILLIS), null);
     }
 
 
@@ -293,8 +281,7 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover, OffsetDateTime.now()
+        int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-                .truncatedTo(ChronoUnit.MILLIS));
         if (updateCount == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -307,8 +294,7 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now()
+        int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-                .truncatedTo(ChronoUnit.MILLIS));
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -321,8 +307,7 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now()
+        int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-                .truncatedTo(ChronoUnit.MILLIS));
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -332,18 +317,17 @@ public class FileStatusPersistenceService {
     @Transactional
     public void overwriteFile(String fileId, String uploader, String filename, boolean keepManualRedactions) {
 
-        int countUpdate = 0;
+        int countUpdate;
 
         if (keepManualRedactions) {
             countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, OffsetDateTime.now()
                     .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         } else {
 
-            fileRepository.findById(fileId).ifPresent((file) -> {
+            fileRepository.findById(fileId).ifPresent((file) -> file.setExcludedPages(new HashSet<>()));
-                file.setExcludedPages(new HashSet<>());
-            });
 
-            countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+            countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, WorkflowStatus.NEW, OffsetDateTime.now()
+                    .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         }
 
         deleteFileAttributes(fileId);
@@ -354,33 +338,13 @@ public class FileStatusPersistenceService {
     }
 
 
-    private List<FileAttributeEntity> convertFileAttributes(String dossierId, FileEntity file,
-                                                            Map<String, String> fileAttributesMap) {
-
-        var dossier = dossierService.getAndValidateDossier(dossierId);
-        List<FileAttributeConfigEntity> configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId());
-
-        return fileAttributesMap.entrySet().stream().map(entry -> {
-            var fa = new FileAttributeEntity();
-            var id = new FileAttributeEntity.FileAttributeEntityId();
-            id.setFileId(file.getId());
-            id.setFileAttributeConfigId(configuration.stream()
-                    .filter(c -> c.getId().equals(entry.getKey()))
-                    .findAny().map(FileAttributeConfigEntity::getId)
-                    .orElseThrow(() -> new BadRequestException("Invalid File Attribute Id")));
-            fa.setFileAttributeId(id);
-            fa.setValue(StringUtils.isBlank(entry.getValue()) ? null : entry.getValue());
-            return fa;
-        }).collect(Collectors.toList());
-    }
-
-
     @Transactional
     public void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime changeDate) {
 
         fileRepository.setLastRedactionModificationDateForFile(fileId, changeDate, OffsetDateTime.now());
     }
 
+
     @Transactional
     public void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate) {
 
@@ -399,6 +363,7 @@ public class FileStatusPersistenceService {
         return fileRepository.countSoftDeletedFilesPerDossierTemplateId(dossierTemplateId);
     }
 
+
     public int countSoftDeletedFilesPerDossierId(String dossierId) {
 
         return fileRepository.countSoftDeletedFilesPerDossierId(dossierId);
@@ -424,4 +389,48 @@ public class FileStatusPersistenceService {
         fileRepository.updateHasHighlights(fileId, hasHighlights);
     }
 
+
+    private int calculateProcessingErrorCounter(String fileId, ProcessingStatus processingStatus) {
+
+        switch (processingStatus) {
+            case ERROR:
+            case PRE_PROCESSING_FAILED:
+                return fileRepository.findById(fileId).map(FileEntity::getProcessingErrorCounter).orElse(0) + 1;
+
+            case PROCESSED:
+                return 0;
+
+            default:
+                return fileRepository.findById(fileId).map(FileEntity::getProcessingErrorCounter).orElse(0);
+        }
+
+    }
+
+
+    private boolean isFileDeleted(String fileId) {
+
+        return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
+    }
+
+
+    private List<FileAttributeEntity> convertFileAttributes(String dossierId, FileEntity file, Map<String, String> fileAttributesMap) {
+
+        var dossier = dossierService.getAndValidateDossier(dossierId);
+        List<FileAttributeConfigEntity> configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId());
+
+        return fileAttributesMap.entrySet().stream().map(entry -> {
+            var fa = new FileAttributeEntity();
+            var id = new FileAttributeEntity.FileAttributeEntityId();
+            id.setFileId(file.getId());
+            id.setFileAttributeConfigId(configuration.stream()
+                    .filter(c -> c.getId().equals(entry.getKey()))
+                    .findAny()
+                    .map(FileAttributeConfigEntity::getId)
+                    .orElseThrow(() -> new BadRequestException("Invalid File Attribute Id")));
+            fa.setFileAttributeId(id);
+            fa.setValue(StringUtils.isBlank(entry.getValue()) ? null : entry.getValue());
+            return fa;
+        }).collect(Collectors.toList());
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -1,88 +1,73 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
 
+import java.time.OffsetDateTime;
+import java.util.List;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
-import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Modifying;
-import org.springframework.data.jpa.repository.Query;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-
 
 public interface FileRepository extends JpaRepository<FileEntity, String> {
 
     boolean existsByDossierIdAndLastUpdatedIsAfter(String dossierId, OffsetDateTime since);
 
+
     List<FileEntity> findByDossierId(String dossierId);
 
+
     @Modifying
-    @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," +
+    @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + " e.lastUpdated = :lastUpdated " + " where e.id =:fileId")
-            " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," +
+    void updateFlags(String fileId, OffsetDateTime lastUpdated, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments,
-            " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " +
-            " e.lastUpdated = :lastUpdated " +
-            " where e.id =:fileId")
-    void updateFlags(String fileId,
-                     OffsetDateTime lastUpdated,
-                     boolean hasRedactions,
-                     boolean hasHints,
-                     boolean hasImages,
-                     boolean hasSuggestions,
-                     boolean hasComments,
                      boolean hasUpdates);
 
-    @Modifying
-    @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " +
-            "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " +
-            "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " +
-            "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " +
-            "f.lastProcessed = :lastProcessed " +
-            "where f.id = :fileId")
-    void updateProcessingStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion,
-                                long rulesVersion, long legalBasisVersion, long analysisDuration, long dossierDictionaryVersion,
-                                int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber);
 
     @Modifying
-    @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," +
+    @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
-            " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId")
+    void updateProcessingStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion, long rulesVersion, long legalBasisVersion,
+                                long analysisDuration, long dossierDictionaryVersion, int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed,
+                                int analysisNumber, int processingErrorCounter);
+
+
+    @Modifying
+    @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId")
     void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate, boolean excludedFromAutomaticAnalysis);
 
+
     @Modifying
-    @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " +
+    @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " + " where f.id = :fileId")
-            " where f.id = :fileId")
     void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate);
 
 
-    @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
-            " f.hasHighlights = :hasHighlights, f.fileSize = :fileSize " +
-            " where f.id = :fileId")
-    void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize);
-
-
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.fileSize = :fileSize  where f.id = :fileId")
     void updateFileSize(String fileId, long fileSize);
 
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated " +
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + " f.hasHighlights = :hasHighlights, f.fileSize = :fileSize, f.processingErrorCounter = :processingErrorCounter " + " where f.id = :fileId")
-            "where f.id = :fileId")
+    void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize, int processingErrorCounter);
-    void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated);
+
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed " +
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
-            "where f.id = :fileId")
+    void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter);
-    void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed);
+
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
-            "f.lastIndexed = :lastIndexed where f.id = :fileId")
+    void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int processingErrorCounter);
-    void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed);
+
+
+    @Modifying(clearAutomatically = true)
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId")
+    void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed, int processingErrorCounter);
 
 
     @Modifying
@@ -107,117 +92,91 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
     @Modifying
     @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id = :fileId")
-    int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
+    int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime softDeletedTime);
-                      OffsetDateTime softDeletedTime);
+
 
     @Modifying
-    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
+    @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.hardDeletedTime = :hardDeletedTime, " + "f.deleted = case " + "when f.deleted is null then :deleted " + "when f.deleted is not null then f.deleted " + "end " + "where f.id = :fileId")
-            "f.hardDeletedTime = :hardDeletedTime, " +
+    int setHardDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime hardDeletedTime, OffsetDateTime deleted);
-            "f.deleted = case " +
+
-            "when f.deleted is null then :deleted " +
-            "when f.deleted is not null then f.deleted " +
-            "end " +
-            "where f.id = :fileId")
-    int setHardDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
-                      OffsetDateTime hardDeletedTime, OffsetDateTime deleted);
 
     @Modifying
-    @Query("update FileEntity f set f.assignee = :assignee, f.lastReviewer = :lastReviewer, f.lastApprover = :lastApprover, " +
+    @Query("update FileEntity f set f.assignee = :assignee, f.lastReviewer = :lastReviewer, f.lastApprover = :lastApprover, " + "f.lastUpdated = :lastUpdated where f.id = :fileId")
-            "f.lastUpdated = :lastUpdated where f.id = :fileId")
     int setAssignee(String fileId, String assignee, String lastReviewer, String lastApprover, OffsetDateTime lastUpdated);
 
+
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.excluded = :excluded, f.lastUpdated = :lastUpdated where f.id = :fileId")
     int toggleExclusion(String fileId, boolean excluded, OffsetDateTime lastUpdated);
 
+
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis, f.lastUpdated = :lastUpdated where f.id = :fileId")
     int toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis, OffsetDateTime lastUpdated);
 
 
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, "
+    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, f.processingErrorCounter = 0 where f.id = :fileId")
-            + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, "
-            + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, "
-            + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
-            + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
-            + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
-            + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false " + "where f.id = :fileId")
     int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded,
                       OffsetDateTime lastUpdated);
 
+
     @Modifying(clearAutomatically = true)
-    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, "
+    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0 where f.id = :fileId")
-            + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, "
-            + "f.lastOCRTime = null, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, "
-            + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
-            + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
-            + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
-            + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false " + "where f.id = :fileId")
     int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded,
                                              OffsetDateTime lastUpdated);
 
-    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" +
+
-            " and ((f.deleted is not null and f.hardDeletedTime is null) or " +
+    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
-            " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
     int countSoftDeletedFilesPerDossierTemplateId(String dossierTemplateId);
 
-    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.id = :dossierId" +
+
-            " and ((f.deleted is not null and f.hardDeletedTime is null) or " +
+    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.id = :dossierId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
-            " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")
     int countSoftDeletedFilesPerDossierId(String dossierId);
 
+
     @Query("select distinct f.dossierId from FileEntity f where f.lastUpdated > :since")
     List<String> findDossierChangeByLastUpdatedIsAfter(OffsetDateTime since);
 
+
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.redactionModificationDate = :redactionModificationDate, f.lastUpdated = :lastUpdated where f.id = :fileId")
     void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime redactionModificationDate, OffsetDateTime lastUpdated);
 
+
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.lastManualChangeDate = :lastManualChangeDate, f.lastUpdated = :lastUpdated where f.id = :fileId")
     void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate, OffsetDateTime lastUpdated);
 
-    @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " +
+
-            "and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'ERROR' )" +
+    @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + "and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'ERROR' )" + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and f.deleted is null and f.hardDeletedTime is null")
-            "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " +
-            "and f.deleted is null and f.hardDeletedTime is null")
     List<FileEntity> getAllRelevantStatusesForReanalysisScheduler();
 
+
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.lastFileAttributeChange = :date, f.lastUpdated = :date where f.id = :fileId")
     void updateLastAttributeChangeDate(String fileId, OffsetDateTime date);
 
+
     @Query("select f from FileEntity f where f.deleted is not null and f.hardDeletedTime is null and f.dossierId in :dossierIds")
     List<FileEntity> getSoftDeletedFiles(List<String> dossierIds);
 
-    @Query("select f.processingStatus as processingStatus, count(f)  as count  from FileEntity f " +
+
-            "inner join DossierEntity d on d.id = f.dossierId " +
+    @Query("select f.processingStatus as processingStatus, count(f)  as count  from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.processingStatus ")
-            "where f.deleted is null and f.hardDeletedTime is null " +
-            "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " +
-            "and d.dossierTemplateId = :dossierTemplateId " +
-            "group by f.processingStatus ")
     List<FileProcessingStatusProjection> countFilesByProcessingStatus(String dossierTemplateId);
 
-    @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " +
+
-            "inner join DossierEntity d on d.id = f.dossierId " +
+    @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.workflowStatus ")
-            "where f.deleted is null and f.hardDeletedTime is null " +
-            "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " +
-            "and d.dossierTemplateId = :dossierTemplateId " +
-            "group by f.workflowStatus ")
     List<FileWorkflowStatusProjection> countFilesByWorkflowStatus(String dossierTemplateId);
 
-    @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " +
+
-            " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " +
+    @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " + " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true)
-            " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" +
-            " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true)
     FilePageCountsProjection countPages(String dossierTemplateId);
 
-    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " +
+
-            "f.hardDeletedTime is null and f.deleted is null and " +
+    @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + "f.hardDeletedTime is null and f.deleted is null and " + "d.dossierTemplateId = :dossierTemplateId and " + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null")
-            "d.dossierTemplateId = :dossierTemplateId and " +
-            "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null")
     int countActiveFiles(String dossierTemplateId);
+
 }
 
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java

@@ -1,8 +1,21 @@
 package com.iqser.red.service.peristence.v1.server.controller;
 
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Sets;
-import com.iqser.red.service.pdftron.redaction.v1.api.model.*;
+import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
 import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
@@ -15,24 +28,12 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.DeleteImportedRedactionsRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
 import com.iqser.red.service.persistence.service.v1.api.resources.ReanalysisResource;
+
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.time.OffsetDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 @Slf4j
 @RestController
@@ -48,8 +49,7 @@ public class ReanalysisController implements ReanalysisResource {
 
 
     @Override
-    public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
+    public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
-                                 @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
 
         var relevantFiles = getAllFilesForDossier(dossierId, true);
         reanalyseFiles(dossierId, force, relevantFiles);
@@ -94,8 +94,7 @@ public class ReanalysisController implements ReanalysisResource {
             fileStatusService.setStatusOcrProcessing(dossierId, fileId);
         } else {
             FileModel dossierFile = fileStatusService.getStatus(fileId);
-            if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus()
+            if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
-                    .equals(WorkflowStatus.APPROVED)) {
                 throw new ConflictException("Cannot analyse a deleted/approved file");
             }
             if (dossierFile.getLastOCRTime() != null) {
@@ -107,6 +106,7 @@ public class ReanalysisController implements ReanalysisResource {
 
 
     public void importRedactions(@RequestBody ByteContentDocument documentRequest) {
+
         try {
             pDFTronRedactionClient.importRedactions(documentRequest);
         } catch (Exception e) {
@@ -129,8 +129,7 @@ public class ReanalysisController implements ReanalysisResource {
 
 
     public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId,
-                        @RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex,
+                        @RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, @RequestBody Set<String> fileIds) {
-                        @RequestBody Set<String> fileIds) {
 
         indexingService.reindex(dossierId, fileIds, dropIndex);
     }
@@ -141,8 +140,7 @@ public class ReanalysisController implements ReanalysisResource {
         boolean hasTextHighlights = pDFTronRedactionClient.convertTextHighlights(textHighlightRequest);
 
         fileStatusService.updateHasHighlights(textHighlightRequest.getFileId(), hasTextHighlights);
-        fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now()
+        fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-                .truncatedTo(ChronoUnit.MILLIS));
 
         if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) {
             fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false);

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java

@@ -1,17 +1,17 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
+import org.springframework.retry.support.RetryTemplate;
+import org.springframework.web.bind.annotation.RestController;
+
 import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
 import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.search.v1.model.IndexMessageType;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.retry.support.RetryTemplate;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RestController;
 
 @Slf4j
 @RestController
@@ -43,7 +43,7 @@ public class FileStatusProcessingUpdateService {
                     fileStatusService.setStatusAnalyse(dossierId, fileId, false);
 
                     //TODO This might be also priority depending on what was the pervious call.
-                    fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null);
+                    fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null, false);
                 }
                 break;
 
@@ -78,14 +78,12 @@ public class FileStatusProcessingUpdateService {
     }
 
 
-
     public void preprocessingFailed(String dossierId, String fileId) {
 
         fileStatusService.updateProcessingStatusPreprocessingFailed(dossierId, fileId);
     }
 
 
-
     public void ocrSuccessful(String dossierId, String fileId) {
 
         retryTemplate.execute(retryContext -> {
@@ -111,16 +109,6 @@ public class FileStatusProcessingUpdateService {
     }
 
 
-    private void setStatusError(String dossierId, String fileId, String reason) {
-
-        retryTemplate.execute(retryContext -> {
-            log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason);
-            fileStatusService.setStatusError(fileId);
-            return null;
-        });
-    }
-
-
     public void indexing(String dossierId, String fileId) {
 
         retryTemplate.execute(retryContext -> {
@@ -146,4 +134,14 @@ public class FileStatusProcessingUpdateService {
         setStatusError(dossierId, fileId, "indexingFailed");
     }
 
+
+    private void setStatusError(String dossierId, String fileId, String reason) {
+
+        retryTemplate.execute(retryContext -> {
+            log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason);
+            fileStatusService.setStatusError(fileId);
+            return null;
+        });
+    }
+
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java

@@ -1,5 +1,19 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
+import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
+
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.transaction.Transactional;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Sets;
@@ -11,7 +25,6 @@ import com.iqser.red.service.peristence.v1.server.model.Pdf2ImageServiceRequest;
 import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
 import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
 import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
@@ -20,7 +33,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
@@ -28,21 +47,10 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.MessageType;
+
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import javax.transaction.Transactional;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
 
 @Slf4j
 @Service
@@ -75,10 +83,7 @@ public class FileStatusService {
 
         var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler();
         var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
-        return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList)
+        return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList());
-                .stream()
-                .filter(FileModel::isAnalysisRequired)
-                .collect(Collectors.toList());
     }
 
 
@@ -120,7 +125,7 @@ public class FileStatusService {
     public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) {
 
         fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize);
-        addToAnalysisQueue(dossierId, fileId, false, Set.of());
+        addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
 
         if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) {
             addToPdf2ImageQueue(dossierId, fileId);
@@ -182,8 +187,7 @@ public class FileStatusService {
 
 
     @Transactional
-    public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse,
+    public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually) {
-                                   boolean triggeredManually) {
 
         log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
 
@@ -199,8 +203,13 @@ public class FileStatusService {
             return;
         }
 
+        if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries() && !triggeredManually) {
+            log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter());
+            return;
+        }
+
         fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS);
-        addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse);
+        addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse, triggeredManually);
     }
 
 
@@ -217,8 +226,7 @@ public class FileStatusService {
 
 
     @Transactional
-    public void setStatusFullReprocess(String dossierId, String fileId, boolean priority,
+    public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) {
-                                       boolean requiresStructureAnalysis) {
 
         FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
 
@@ -227,24 +235,25 @@ public class FileStatusService {
             return;
         }
 
+        if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) {
+            log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter());
+            return;
+        }
+
         if (requiresStructureAnalysis) {
             log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
             fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
             fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
         }
 
-        addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
+        addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false);
     }
 
 
     @SneakyThrows
     public void addToPreprocessingQueue(String dossierId, String fileId, String filename) {
 
-        var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder()
+        var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder().dossierId(dossierId).fileId(fileId).fileName(filename).build();
-                .dossierId(dossierId)
-                .fileId(fileId)
-                .fileName(filename)
-                .build();
 
         setStatusPreProcessing(fileId);
 
@@ -253,8 +262,7 @@ public class FileStatusService {
 
 
     @Transactional
-    protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority,
+    protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually) {
-                                      Set<Integer> sectionsToReanalyse) {
 
         var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
         var fileEntity = fileStatusPersistenceService.getStatus(fileId);
@@ -264,6 +272,11 @@ public class FileStatusService {
             return;
         }
 
+        if (fileEntity.getProcessingErrorCounter() >= settings.getMaxErrorRetries() && !triggeredManually) {
+            log.warn("File {} was {} times retried with failure", fileEntity.getId(), fileEntity.getProcessingErrorCounter());
+            return;
+        }
+
         var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
         reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
 
@@ -316,28 +329,6 @@ public class FileStatusService {
     }
 
 
-    private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
-                                             ProcessingStatus processingStatus, FileModel fileModel) {
-
-        if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
-            return MessageType.STRUCTURE_ANALYSE;
-        }
-        if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
-            return MessageType.ANALYSE;
-        }
-        if (reanalyse) {
-            if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed()
-                    .isBefore(fileModel.getLastFileAttributeChange())) {
-                return MessageType.ANALYSE;
-            }
-            return MessageType.REANALYSE;
-        } else if(manualRedactionProviderService.getManualRedactions(fileId).getResizeRedactions().stream().filter(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed())).findFirst().isPresent()){
-            return MessageType.REANALYSE;
-        }
-        return MessageType.ANALYSE;
-    }
-
-
     @Transactional
     public void setStatusReprocess(String dossierId, String fileId, boolean priority) {
 
@@ -451,6 +442,11 @@ public class FileStatusService {
             return;
         }
 
+        if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) {
+            log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter());
+            return;
+        }
+
         fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING);
         addToOcrQueue(dossierId, fileId, 2);
     }
@@ -493,19 +489,6 @@ public class FileStatusService {
     }
 
 
-    private void addToOcrQueue(String dossierId, String fileId, int priority) {
-
-        try {
-            rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new DocumentRequest(dossierId, fileId)), message -> {
-                message.getMessageProperties().setPriority(priority);
-                return message;
-            });
-        } catch (JsonProcessingException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-
     public void updateLastOCRTime(String fileId) {
 
         fileStatusPersistenceService.updateLastOCRTime(fileId, OffsetDateTime.now());
@@ -548,8 +531,7 @@ public class FileStatusService {
 
         // wipe force redactions
         var forceRedactions = forceRedactionPersistenceService.findForceRedactions(fileId, false);
-        forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId()
+        forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
-                .getAnnotationId(), now));
 
         // wipe add manual redactions
         var addRedactions = addRedactionPersistenceService.findAddRedactions(fileId, false);
@@ -557,30 +539,108 @@ public class FileStatusService {
 
         // wipe removeRedactions
         var removeRedactions = removeRedactionPersistenceService.findRemoveRedactions(fileId, false);
-        removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId()
+        removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
-                .getAnnotationId(), now));
 
         // wipe image recat
         var imageRecategorizations = imageRecategorizationPersistenceService.findRecategorizations(fileId, false);
-        imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId()
+        imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));// wipe image recat
-                .getAnnotationId(), now));// wipe image recat
 
         // wipe resize redactions
         var resizeRedactions = resizeRedactionPersistenceService.findResizeRedactions(fileId, false);
-        resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId()
+        resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
-                .getAnnotationId(), now));
 
         // wipe legal basis changes
         var legalBasisChanges = legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false);
-        legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId()
+        legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
-                .getAnnotationId(), now));
 
         fileStatusPersistenceService.updateHasComments(fileId, false);
     }
 
 
-    private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(
+    @Transactional
-            List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
+    public void setStatusAnalyse(String dossierId, String fileId, boolean priority) {
+
+        FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
+
+        if (fileStatus.isExcluded()) {
+            log.debug("File {} is excluded", fileStatus.getId());
+            return;
+        }
+
+        if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) {
+            log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter());
+            return;
+        }
+
+        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE);
+        addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false);
+    }
+
+
+    public int countSoftDeletedFiles(String dossierId) {
+
+        return fileStatusPersistenceService.countSoftDeletedFilesPerDossierId(dossierId);
+    }
+
+
+    protected void addToNerQueue(String dossierId, String fileId) {
+
+        setStatusNerAnalyzing(fileId);
+        try {
+            rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(NerServiceRequest.builder()
+                    .dossierId(dossierId)
+                    .fileId(fileId)
+                    .targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION)
+                    .responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION)
+                    .build()), message -> {
+                message.getMessageProperties().setPriority(1);
+                return message;
+            });
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) {
+
+        if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
+            return MessageType.STRUCTURE_ANALYSE;
+        }
+        if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
+            return MessageType.ANALYSE;
+        }
+        if (reanalyse) {
+            if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) {
+                return MessageType.ANALYSE;
+            }
+            return MessageType.REANALYSE;
+        } else if (manualRedactionProviderService.getManualRedactions(fileId)
+                .getResizeRedactions()
+                .stream()
+                .filter(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed()))
+                .findFirst()
+                .isPresent()) {
+            return MessageType.REANALYSE;
+        }
+        return MessageType.ANALYSE;
+    }
+
+
+    private void addToOcrQueue(String dossierId, String fileId, int priority) {
+
+        try {
+            rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new DocumentRequest(dossierId, fileId)), message -> {
+                message.getMessageProperties().setPriority(priority);
+                return message;
+            });
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
 
         List<com.iqser.red.service.redaction.v1.model.FileAttribute> fileAttributeList = new ArrayList<>();
 
@@ -611,44 +671,4 @@ public class FileStatusService {
         return fileStatusPersistenceService.hasChangesSince(dossierId, since);
     }
 
-
-    @Transactional
-    public void setStatusAnalyse(String dossierId, String fileId, boolean priority) {
-
-        FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
-
-        if (fileStatus.isExcluded()) {
-            log.debug("File {} is excluded", fileStatus.getId());
-            return;
-        }
-
-        fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE);
-        addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet());
-    }
-
-
-    protected void addToNerQueue(String dossierId, String fileId) {
-
-        setStatusNerAnalyzing(fileId);
-        try {
-            rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(NerServiceRequest.builder()
-                    .dossierId(dossierId)
-                    .fileId(fileId)
-                    .targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION)
-                    .responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION)
-                    .build()), message -> {
-                message.getMessageProperties().setPriority(1);
-                return message;
-            });
-        } catch (JsonProcessingException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-
-    public int countSoftDeletedFiles(String dossierId) {
-
-        return fileStatusPersistenceService.countSoftDeletedFilesPerDossierId(dossierId);
-    }
-
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java

@@ -30,4 +30,6 @@ public class FileManagementServiceSettings {
 
     private boolean pdf2ImageServiceEnabled;
 
+    private int maxErrorRetries = 5;
+
 }

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/33-add-file-processing-error-counter-column.changelog.yaml

@@ -0,0 +1,11 @@
+databaseChangeLog:
+  - changeSet:
+      id: add-file-processing-error-counter-column
+      author: philipp
+      changes:
+        - addColumn:
+            columns:
+              - column:
+                  name: processing_error_counter
+                  type: BIGINT
+            tableName: file

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -77,3 +77,7 @@ databaseChangeLog:
       file: db/changelog/sql/31-watermark-configuration.sql
   - include:
       file: db/changelog/32-added-skipped-color-type-table.changelog.yaml
+  - include:
+      file: db/changelog/33-add-file-processing-error-counter-column.changelog.yaml
+  - include:
+      file: db/changelog/sql/33-set-file-processing-error-counter.sql

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/33-set-file-processing-error-counter.sql

@@ -0,0 +1 @@
+update file set processing_error_counter = 0 where processing_error_counter is null;