Pull request #198: RED-3242

Merge in RED/persistence-service from RED-3242 to master

* commit 'a2066533e286e8a92c9900a15aa1430c40fe81a2':
  Fixded PMD
  Fixded PMD
  Final touches for RED-3242
  RED-3242 - excluded from automatic analysis

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/BaseAnnotation.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.persistence.service.v1.api.model.annotations;
+
+import lombok.Data;
+
+import java.time.OffsetDateTime;
+
+@Data
+public class BaseAnnotation {
+
+    private String annotationId;
+    private String fileId;
+    private String user;
+    private AnnotationStatus status;
+    private OffsetDateTime requestDate;
+    private OffsetDateTime processedDate;
+    private OffsetDateTime softDeletedTime;
+}

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/IdRemoval.java

@@ -5,21 +5,12 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.time.OffsetDateTime;
-
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
-public class IdRemoval {
+public class IdRemoval extends BaseAnnotation {
 
-    private String annotationId;
-    private String fileId;
-    private String user;
-    private AnnotationStatus status;
     private boolean removeFromDictionary;
-    private OffsetDateTime requestDate;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime softDeletedTime;
 
 }

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualForceRedaction.java

@@ -5,21 +5,12 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.time.OffsetDateTime;
-
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
-public class ManualForceRedaction {
+public class ManualForceRedaction extends BaseAnnotation {
 
-    private String annotationId;
-    private String fileId;
-    private String user;
-    private AnnotationStatus status;
     private String legalBasis;
-    private OffsetDateTime requestDate;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime softDeletedTime;
 
 }

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualImageRecategorization.java

@@ -5,20 +5,11 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.time.OffsetDateTime;
-
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
-public class ManualImageRecategorization {
+public class ManualImageRecategorization extends BaseAnnotation {
 
-    private String annotationId;
-    private String fileId;
-    private String user;
-    private AnnotationStatus status;
     private String type;
-    private OffsetDateTime requestDate;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime softDeletedTime;
 }

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualLegalBasisChange.java

@@ -5,23 +5,14 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.time.OffsetDateTime;
-
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
-public class ManualLegalBasisChange {
+public class ManualLegalBasisChange extends BaseAnnotation {
 
-    private String annotationId;
-    private String fileId;
-    private String user;
-    private AnnotationStatus status;
     private String section;
     private String value;
     private String legalBasis;
-    private OffsetDateTime requestDate;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime softDeletedTime;
 
 }

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualRedactionEntry.java

@@ -5,7 +5,6 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -13,23 +12,16 @@ import java.util.List;
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
-public class ManualRedactionEntry {
+public class ManualRedactionEntry extends BaseAnnotation {
 
-    private String annotationId;
-    private String fileId;
-    private String user;
     private String type;
     private String value;
     private String reason;
     private String legalBasis;
-    private AnnotationStatus status;
     private String section;
     private boolean rectangle;
     private boolean addToDictionary;
     private boolean addToDossierDictionary;
-    private OffsetDateTime requestDate;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime softDeletedTime;
     private List<Rectangle> positions = new ArrayList<>();
     private String textBefore;
     private String textAfter;

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualResizeRedaction.java

@@ -1,27 +1,19 @@
 package com.iqser.red.service.persistence.service.v1.api.model.annotations;
 
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.List;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+import java.util.ArrayList;
+import java.util.List;
+
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
-public class ManualResizeRedaction {
+public class ManualResizeRedaction extends BaseAnnotation {
 
-    private String annotationId;
-    private String fileId;
-    private String user;
-    private AnnotationStatus status;
-    private OffsetDateTime requestDate;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime softDeletedTime;
     private String value;
     @Builder.Default
     private List<Rectangle> positions = new ArrayList<>();

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java

@@ -21,29 +21,31 @@ public interface ReanalysisResource {
 
     @PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM)
     void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                          @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
                           @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force);
 
 
-    @PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_PARAM)
-    void reanalyzeFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
-                       @RequestBody Set<Integer> sectionsToReanalyse);
-
-
     @PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + BULK_REST_PATH)
     void reanalyzeFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds,
+                        @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
                         @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force);
 
 
     @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM)
-    void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId);
+    void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                    @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser);
 
 
     @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_PARAM)
-    void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force);
+    void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
+                 @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
+                 @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force);
 
 
     @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + BULK_REST_PATH)
-    void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds);
+    void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                  @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
+                  @RequestBody Set<String> fileIds);
 
 
     @PostMapping(value = REINDEX_PATH)

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java

@@ -24,6 +24,8 @@ public interface StatusResource {
 
     String EXCLUDED_STATUS_PARAM = "excluded";
 
+    String EXCLUDED_FROM_AUTOMATIC_ANALYSIS_PARAM = "excludedFromAutomaticAnalysis";
+
     String APPROVER_ID_REQUEST_PARAM = "approverId";
     String ASSIGNEE_ID_REQUEST_PARAM = "assigneeId";
     String USER_ID_REQUEST_PARAM = "userId";
@@ -74,6 +76,9 @@ public interface StatusResource {
     @PostMapping(value = STATUS_PATH + "/toggle-analysis" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE)
     void toggleExclusion(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestParam(EXCLUDED_STATUS_PARAM) boolean excluded);
 
+    @PostMapping(value = STATUS_PATH + "/toggle-automatic-analysis" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE)
+    void toggleAutomaticAnalysis(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestParam(EXCLUDED_STATUS_PARAM) boolean excludedFromAutomaticAnalysis);
+
     @PostMapping(value = STATUS_PATH + "/exclude-pages" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE)
     void excludePages(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set<Integer> pages);
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IBaseAnnotation.java

@@ -0,0 +1,24 @@
+package com.iqser.red.service.persistence.management.v1.processor.entity.annotations;
+
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+
+import java.time.OffsetDateTime;
+
+public interface IBaseAnnotation {
+
+    AnnotationEntityId getId();
+
+    AnnotationStatus getStatus();
+
+    OffsetDateTime getRequestDate();
+
+    void setRequestDate(OffsetDateTime requestedDate);
+
+    OffsetDateTime getProcessedDate();
+
+    void setProcessedDate(OffsetDateTime processedDate);
+
+    OffsetDateTime getSoftDeletedTime();
+
+    void setSoftDeletedTime(OffsetDateTime softDeletedTime);
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java

@@ -16,7 +16,7 @@ import java.time.OffsetDateTime;
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
-public class IdRemovalEntity {
+public class IdRemovalEntity implements IBaseAnnotation {
 
     @EmbeddedId
     private AnnotationEntityId id;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java

@@ -16,7 +16,7 @@ import java.time.OffsetDateTime;
 @NoArgsConstructor
 @Entity
 @Table(name = "manual_force_redaction")
-public class ManualForceRedactionEntity {
+public class ManualForceRedactionEntity implements IBaseAnnotation {
 
     @EmbeddedId
     private AnnotationEntityId id;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualImageRecategorizationEntity.java

@@ -16,7 +16,7 @@ import java.time.OffsetDateTime;
 @NoArgsConstructor
 @Entity
 @Table(name = "manual_image_recategorization")
-public class ManualImageRecategorizationEntity {
+public class ManualImageRecategorizationEntity implements IBaseAnnotation {
 
     @EmbeddedId
     private AnnotationEntityId id;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java

@@ -16,7 +16,7 @@ import java.time.OffsetDateTime;
 @NoArgsConstructor
 @Entity
 @Table(name = "manual_legal_basis_change")
-public class ManualLegalBasisChangeEntity {
+public class ManualLegalBasisChangeEntity implements IBaseAnnotation {
 
     @EmbeddedId
     private AnnotationEntityId id;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java

@@ -18,7 +18,7 @@ import java.util.List;
 @AllArgsConstructor
 @NoArgsConstructor
 @Table(name = "manual_redaction")
-public class ManualRedactionEntryEntity {
+public class ManualRedactionEntryEntity implements IBaseAnnotation {
 
     @EmbeddedId
     private AnnotationEntityId id;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java

@@ -1,33 +1,24 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.annotations;
 
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.persistence.Column;
-import javax.persistence.ElementCollection;
-import javax.persistence.EmbeddedId;
-import javax.persistence.Entity;
-import javax.persistence.EnumType;
-import javax.persistence.Enumerated;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
-
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+import javax.persistence.*;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
 @Table(name = "manual_resize_redaction")
-public class ManualResizeRedactionEntity {
+public class ManualResizeRedactionEntity implements IBaseAnnotation {
 
     @EmbeddedId
     private AnnotationEntityId id;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java

@@ -118,6 +118,9 @@ public class FileEntity {
     @Column
     private boolean excluded;
 
+    @Column
+    private boolean excludedFromAutomaticAnalysis;
+
     @Column
     private OffsetDateTime hardDeletedTime;
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -71,7 +71,7 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateFlags(fileId, OffsetDateTime.now(), hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates); 
+        fileRepository.updateFlags(fileId, OffsetDateTime.now(), hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates);
     }
 
 
@@ -81,9 +81,8 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateWorkflowStatus(fileId, workflowStatus,
+        fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
+                approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null, approval);
-                approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null); 
 
     }
 
@@ -97,9 +96,9 @@ public class FileStatusPersistenceService {
         if (processingStatus == ProcessingStatus.PROCESSED) {
             // In case the file is updated to "processed", "lastProcessed" date should be updated to "now"
             fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                    OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); 
+                    OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         } else {
-            fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); 
+            fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         }
     }
 
@@ -112,7 +111,7 @@ public class FileStatusPersistenceService {
         }
         fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED,
                 OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); 
+                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
     }
 
 
@@ -122,7 +121,7 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateLastOCRTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), time);  
+        fileRepository.updateLastOCRTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), time);
     }
 
 
@@ -132,7 +131,7 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateHasComments(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasComments); 
+        fileRepository.updateHasComments(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasComments);
     }
 
 
@@ -142,7 +141,7 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        fileRepository.updateLastManualRedaction(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date); 
+        fileRepository.updateLastManualRedaction(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date);
     }
 
 
@@ -154,7 +153,7 @@ public class FileStatusPersistenceService {
             return;
         }
         fileRepository.setUpdateLastManualRedactionAndHasSuggestions(fileId,
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions); 
+                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions);
     }
 
 
@@ -171,7 +170,7 @@ public class FileStatusPersistenceService {
             var fileAttributeEntities = convertFileAttributes(dossierId, file, fileAttributes);
             file.setLastFileAttributeChange(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
             file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-            file.setFileAttributes(fileAttributeEntities); 
+            file.setFileAttributes(fileAttributeEntities);
         }, () -> {
             throw new NotFoundException("Unknown file=" + fileId);
         });
@@ -188,7 +187,7 @@ public class FileStatusPersistenceService {
 
             file.setLastManualRedaction(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
             file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-            file.setExcludedPages(excludedPages); 
+            file.setExcludedPages(excludedPages);
         }, () -> {
             throw new NotFoundException("Unknown file=" + fileId);
         });
@@ -224,7 +223,7 @@ public class FileStatusPersistenceService {
     public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
 
         int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.DELETED,
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime); 
+                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -237,7 +236,7 @@ public class FileStatusPersistenceService {
         int countUpdate = fileRepository.setHardDelete(fileId, ProcessingStatus.DELETED,
                 OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
                 OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); 
+                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -279,7 +278,18 @@ public class FileStatusPersistenceService {
         if (isFileDeleted(fileId)) {
             return;
         }
-        int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); 
+        int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+        if (countUpdate == 0) {
+            throw new NotFoundException("Unknown file=" + fileId);
+        }
+    }
+
+    @Transactional
+    public void toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis) {
+        if (isFileDeleted(fileId)) {
+            return;
+        }
+        int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }
@@ -296,7 +306,7 @@ public class FileStatusPersistenceService {
         int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS,
                 WorkflowStatus.NEW,
                 OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); 
+                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
@@ -16,9 +16,9 @@ import javax.transaction.Transactional;
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 @Service
 @RequiredArgsConstructor
@@ -28,7 +28,7 @@ public class AddRedactionPersistenceService {
     private final ManualRedactionRepository manualRedactionRepository;
 
 
-    public void insert(String fileId, String annotationId, AddRedactionRequest addRedactionRequest) {
+    public ManualRedactionEntryEntity insert(String fileId, String annotationId, AddRedactionRequest addRedactionRequest) {
 
         ManualRedactionEntryEntity manualRedactionEntry = new ManualRedactionEntryEntity();
         manualRedactionEntry.setId(new AnnotationEntityId(annotationId, fileId));
@@ -37,13 +37,18 @@ public class AddRedactionPersistenceService {
         manualRedactionEntry.setPositions(convert(addRedactionRequest.getPositions()));
         manualRedactionEntry.setTypeId(addRedactionRequest.getTypeId());
 
-        manualRedactionRepository.save(manualRedactionEntry);
+        if (addRedactionRequest.getStatus() == AnnotationStatus.APPROVED) {
+            manualRedactionEntry.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+        }
+
+
+        return manualRedactionRepository.save(manualRedactionEntry);
 
     }
 
 
     @Transactional
-    public void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter){
+    public void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter) {
         manualRedactionRepository.updateSurroundingText(id, textBefore, textAfter);
     }
 
@@ -69,7 +74,7 @@ public class AddRedactionPersistenceService {
 
     public Set<ManualRedactionEntryEntity> findAddRedactions(String fileId, boolean includeDeletions) {
 
-        return manualRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions).stream().collect(Collectors.toSet());
+        return new HashSet<>(manualRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions));
 
     }
 
@@ -94,19 +99,13 @@ public class AddRedactionPersistenceService {
     @Transactional
     public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) {
 
-        manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId),
+        manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus);
-            OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-            annotationStatus);
     }
 
     @Transactional
-    public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary
+    public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary, boolean isAddOrRemoveFromDossierDictionary) {
-            , boolean isAddOrRemoveFromDossierDictionary) {
 
-        manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId),
+        manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus, isAddOrRemoveFromDictionary, isAddOrRemoveFromDossierDictionary);
-                OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
-                annotationStatus,
-                isAddOrRemoveFromDictionary, isAddOrRemoveFromDossierDictionary);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java

@@ -1,12 +1,14 @@
-package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
 
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
+import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import javax.transaction.Transactional;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
@@ -33,17 +35,12 @@ public class ForceRedactionPersistenceService {
         manualForceRedaction.setId(new AnnotationEntityId(forceRedactionRequest.getAnnotationId(), fileId));
         BeanUtils.copyProperties(forceRedactionRequest, manualForceRedaction);
         manualForceRedaction.setRequestDate(OffsetDateTime.now());
-        if(manualForceRedaction.getStatus().equals(AnnotationStatus.APPROVED)) {
-            manualForceRedaction.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-        }
-
         forceRedactionRepository.save(manualForceRedaction);
     }
 
     @Transactional
     public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) {
-        forceRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus,
+        forceRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus);
-                OffsetDateTime.now());
     }
 
 
@@ -72,7 +69,10 @@ public class ForceRedactionPersistenceService {
 
     public Set<ManualForceRedactionEntity> findForceRedactions(String fileId, boolean includeDeletions) {
 
-        return forceRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions).stream().collect(Collectors.toSet());
+        return new HashSet<>(forceRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions));
     }
 
+    public void markAsProcessed(ManualForceRedaction e) {
+        forceRedactionRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+    }
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity;
@@ -6,12 +6,15 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ImageRecategorizationRepository;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
 import lombok.RequiredArgsConstructor;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
 import javax.transaction.Transactional;
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
 
 @Service
@@ -34,8 +37,7 @@ public class ImageRecategorizationPersistenceService {
     @Transactional
     public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) {
 
-        imageRecategorizationRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus,
+        imageRecategorizationRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus);
-                OffsetDateTime.now());
     }
 
     @Transactional
@@ -68,5 +70,7 @@ public class ImageRecategorizationPersistenceService {
 
     }
 
-
+    public void markAsProcessed(ManualImageRecategorization e) {
+        imageRecategorizationRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+    }
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity;
@@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
 
 import javax.transaction.Transactional;
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -22,13 +23,18 @@ public class LegalBasisChangePersistenceService {
     private final LegalBasisChangeRepository legalBasisChangeRepository;
 
 
-    public void insert(String fileId, LegalBasisChangeRequest legalBasisChangeRequest) {
+    public ManualLegalBasisChangeEntity insert(String fileId, LegalBasisChangeRequest legalBasisChangeRequest) {
 
         ManualLegalBasisChangeEntity manualLegalBasisChange = new ManualLegalBasisChangeEntity();
         manualLegalBasisChange.setId(new AnnotationEntityId(legalBasisChangeRequest.getAnnotationId(), fileId));
         BeanUtils.copyProperties(legalBasisChangeRequest, manualLegalBasisChange);
-        manualLegalBasisChange.setRequestDate(OffsetDateTime.now());
+        manualLegalBasisChange.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
-        legalBasisChangeRepository.save(manualLegalBasisChange);
+
+        if(legalBasisChangeRequest.getStatus() == AnnotationStatus.APPROVED) {
+            manualLegalBasisChange.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+        }
+
+        return legalBasisChangeRepository.save(manualLegalBasisChange);
 
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java

@@ -1,10 +1,11 @@
-package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RemoveRedactionRepository;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest;
 import lombok.RequiredArgsConstructor;
 import org.springframework.beans.BeanUtils;
@@ -12,6 +13,7 @@ import org.springframework.stereotype.Service;
 
 import javax.transaction.Transactional;
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -74,5 +76,8 @@ public class RemoveRedactionPersistenceService {
                 annotationStatus, OffsetDateTime.now(), isAddOrRemoveFromDictionary);
     }
 
-
+    @Transactional
+    public void markAsProcessed(IdRemoval e) {
+        removeRedactionRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
+    }
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
+package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations;
 
 import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
 
@@ -29,18 +29,19 @@ public class ResizeRedactionPersistenceService {
 
 
     @Transactional
-    public void insert(String fileId, ResizeRedactionRequest resizeRedactionRequest) {
+    public ManualResizeRedactionEntity insert(String fileId, ResizeRedactionRequest resizeRedactionRequest) {
 
         ManualResizeRedactionEntity manualResizeRedaction = new ManualResizeRedactionEntity();
         manualResizeRedaction.setId(new AnnotationEntityId(resizeRedactionRequest.getAnnotationId(), fileId));
         BeanUtils.copyProperties(resizeRedactionRequest, manualResizeRedaction);
         manualResizeRedaction.setPositions(convert(resizeRedactionRequest.getPositions(), RectangleEntity.class));
         manualResizeRedaction.setRequestDate(OffsetDateTime.now());
+
         if(manualResizeRedaction.getStatus().equals(AnnotationStatus.APPROVED)) {
             manualResizeRedaction.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
         }
 
-        resizeRedactionRepository.save(manualResizeRedaction);
+        return resizeRedactionRepository.save(manualResizeRedaction);
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -45,9 +45,9 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
                                 int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber);
 
     @Modifying
-    @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " +
+    @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," +
-            "where f.id = :fileId")
+            " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId")
-    void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate);
+    void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate, boolean excludedFromAutomaticAnalysis);
 
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated " +
@@ -109,6 +109,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     @Query("update FileEntity f set f.excluded = :excluded, f.lastUpdated = :lastUpdated where f.id = :fileId")
     int toggleExclusion(String fileId, boolean excluded, OffsetDateTime lastUpdated);
 
+    @Modifying(clearAutomatically = true)
+    @Query("update FileEntity f set f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis, f.lastUpdated = :lastUpdated where f.id = :fileId")
+    int toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis, OffsetDateTime lastUpdated);
+
     @Modifying(clearAutomatically = true)
     @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " +
             "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " +

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ForceRedactionRepository.java

@@ -16,9 +16,9 @@ public interface ForceRedactionRepository extends JpaRepository<ManualForceRedac
     List<ManualForceRedactionEntity> findByIdFileId(String fileId);
 
     @Modifying
-    @Query("update ManualForceRedactionEntity mfr set mfr.status = :annotationStatus, mfr.processedDate = :processedDate " +
+    @Query("update ManualForceRedactionEntity mfr set mfr.status = :annotationStatus " +
             "where mfr.id = :annotationEntityId")
-    void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus, OffsetDateTime processedDate);
+    void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus);
 
     @Modifying
     @Query("update ManualForceRedactionEntity mfr set mfr.softDeletedTime = :softDeletedTime " +
@@ -31,4 +31,7 @@ public interface ForceRedactionRepository extends JpaRepository<ManualForceRedac
     @Query("select mfr from ManualForceRedactionEntity mfr where mfr.id.fileId = :fileId and (:includeDeletions = true or mfr.softDeletedTime is null)")
     List<ManualForceRedactionEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions);
 
+    @Modifying
+    @Query("update ManualForceRedactionEntity mfr set mfr.processedDate = :processedDate where mfr.id = :annotationEntityId")
+    void markAsProcessed(AnnotationEntityId annotationEntityId, OffsetDateTime processedDate);
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ImageRecategorizationRepository.java

@@ -15,9 +15,9 @@ public interface ImageRecategorizationRepository extends JpaRepository<ManualIma
     List<ManualImageRecategorizationEntity> findByIdFileId(String fileId);
 
     @Modifying
-    @Query("update ManualImageRecategorizationEntity mir set mir.status = :annotationStatus, mir.processedDate = :processedDate " +
+    @Query("update ManualImageRecategorizationEntity mir set mir.status = :annotationStatus " +
             "where mir.id = :annotationEntityId")
-    void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus, OffsetDateTime processedDate);
+    void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus);
 
     @Modifying
     @Query("update ManualImageRecategorizationEntity mir set mir.softDeletedTime = :softDeletedTime " +
@@ -29,4 +29,8 @@ public interface ImageRecategorizationRepository extends JpaRepository<ManualIma
 
     @Query("select mir from ManualImageRecategorizationEntity mir where mir.id.fileId = :fileId and (:includeDeletions = true or mir.softDeletedTime is null)")
     List<ManualImageRecategorizationEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions);
+
+    @Modifying
+    @Query("update ManualImageRecategorizationEntity mir set mir.processedDate = :processedDate where mir.id = :annotationEntityId")
+    void markAsProcessed(AnnotationEntityId annotationEntityId, OffsetDateTime processedDate);
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/LegalBasisChangeRepository.java

@@ -13,8 +13,6 @@ import java.util.Optional;
 
 public interface LegalBasisChangeRepository extends JpaRepository<ManualLegalBasisChangeEntity, AnnotationEntityId> {
 
-    List<ManualLegalBasisChangeEntity> findByIdFileId(String fileId);
-
     @Modifying
     @Query("update ManualLegalBasisChangeEntity mlbc set mlbc.status = :annotationStatus, mlbc.processedDate = :processedDate " +
             "where mlbc.id = :annotationEntityId")

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ManualRedactionRepository.java

@@ -20,14 +20,13 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
     void updateSoftDelete(AnnotationEntityId id, OffsetDateTime softDeleteTime);
 
     @Modifying
-    @Query("update ManualRedactionEntryEntity m set m.processedDate = :processedDate, m.status = :annotationStatus" +
+    @Query("update ManualRedactionEntryEntity m set m.status = :annotationStatus where m.id = :id")
-            " where m.id = :id")
+    void updateStatus(AnnotationEntityId id, AnnotationStatus annotationStatus);
-    void updateStatus(AnnotationEntityId id, OffsetDateTime processedDate, AnnotationStatus annotationStatus);
 
     @Modifying
-    @Query("update ManualRedactionEntryEntity m set m.processedDate = :processedDate, m.status = :annotationStatus," +
+    @Query("update ManualRedactionEntryEntity m set m.status = :annotationStatus," +
             " m.addToDictionary = :isAddOrRemoveFromDictionary, m.addToDossierDictionary = :isAddOrRemoveFromDossierDictionary where m.id = :id")
-    void updateStatus(AnnotationEntityId id, OffsetDateTime processedDate, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary,
+    void updateStatus(AnnotationEntityId id,  AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary,
             boolean isAddOrRemoveFromDossierDictionary);
 
     @Query("select m from ManualRedactionEntryEntity m where m.id = :id and m.softDeletedTime is null")

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/RemoveRedactionRepository.java

@@ -37,4 +37,7 @@ public interface RemoveRedactionRepository extends JpaRepository<IdRemovalEntity
     @Query("select idr from IdRemovalEntity idr where idr.id.fileId = :fileId and (:includeDeletions = true or idr.softDeletedTime is null)")
     List<IdRemovalEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions);
 
+    @Modifying
+    @Query("update IdRemovalEntity idr set idr.processedDate = :processedDate where idr.id = :annotationEntityId")
+    void markAsProcessed(AnnotationEntityId annotationEntityId, OffsetDateTime processedDate);
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java

@@ -149,6 +149,13 @@ public class FileStatusController implements StatusResource {
 
     }
 
+    public void toggleAutomaticAnalysis(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                                @PathVariable(FILE_ID) String fileId, @RequestParam(EXCLUDED_STATUS_PARAM) boolean excludedFromAutomaticAnalysis) {
+
+        excludeFromAnalysis.toggleAutomaticAnalysis(dossierId, fileId, excludedFromAutomaticAnalysis);
+
+    }
+
 
     public void excludePages(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set<Integer> pages) {
         FileEntity fileStatus = fileStatusService.getStatus(fileId);

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java

@@ -41,51 +41,35 @@ public class ReanalysisController implements ReanalysisResource {
 
     @Override
     public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
+                                 @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
                                  @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
 
-        var relevantFiles = getRelevantFiles(dossierId);
+        var relevantFiles = getRelevantFiles(dossierId, triggeredByUser);
-
         reanalyseFiles(dossierId, force, relevantFiles);
     }
 
 
-    @Override
-    public void reanalyzeFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
-                              @PathVariable(FILE_ID_PARAM) String fileId,
-                              @RequestBody Set<Integer> sectionsToReanalyse) {
-
-        log.info("Reanalyse: {} / {} with sections: {}", dossierId, fileId, sectionsToReanalyse);
-        if (sectionsToReanalyse != null) {
-            var relevantFiles = getRelevantFiles(dossierId, Sets.newHashSet(fileId));
-            relevantFiles.forEach(file -> {
-                fileStatusService.setStatusReprocess(dossierId, fileId, 10, sectionsToReanalyse);
-            });
-        } else {
-            this.reanalyzeFiles(dossierId, Sets.newHashSet(fileId), false);
-        }
-    }
-
-
     @Override
     public void reanalyzeFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds,
+                               @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
                                @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
 
-        var relevantFiles = getRelevantFiles(dossierId, fileIds);
+        var relevantFiles = getRelevantFiles(dossierId, fileIds, triggeredByUser);
         reanalyseFiles(dossierId, force, relevantFiles);
     }
 
 
-    public void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId) {
+    public void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser) {
 
-        var relevantFiles = getRelevantFiles(dossierId);
+        var relevantFiles = getRelevantFiles(dossierId, triggeredByUser);
 
         relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId()));
     }
 
 
-    public void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds) {
+    public void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, @RequestBody Set<String> fileIds) {
 
-        var relevantFiles = getRelevantFiles(dossierId, fileIds);
+        var relevantFiles = getRelevantFiles(dossierId, fileIds, triggeredByUser);
 
         relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId()));
     }
@@ -96,7 +80,9 @@ public class ReanalysisController implements ReanalysisResource {
     }
 
 
-    public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
+    public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
+                        @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser,
+                        @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
 
         dossierPersistenceService.getAndValidateDossier(dossierId);
 
@@ -110,26 +96,27 @@ public class ReanalysisController implements ReanalysisResource {
             if (dossierFile.getLastOCRTime() != null) {
                 throw new ConflictException("File already has been OCR processed");
             }
-            ocrFiles(dossierId, Sets.newHashSet(fileId));
+            ocrFiles(dossierId,triggeredByUser, Sets.newHashSet(fileId));
         }
     }
 
 
-    private List<FileEntity> getRelevantFiles(String dossierId) {
+    private List<FileEntity> getRelevantFiles(String dossierId, boolean triggeredByUser) {
 
         var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
         return fileStatusService.getDossierStatus(dossier.getId())
                 .stream()
                 .filter(fileStatus -> !fileStatus.getProcessingStatus().equals(ProcessingStatus.DELETED))
                 .filter(fileStatus -> !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED))
+                .filter(fileStatus -> triggeredByUser || !fileStatus.isExcludedFromAutomaticAnalysis())
                 .collect(Collectors.toList());
 
     }
 
 
-    private List<FileEntity> getRelevantFiles(String dossierId, Collection<String> fileIds) {
+    private List<FileEntity> getRelevantFiles(String dossierId, Collection<String> fileIds, boolean triggeredByUser) {
 
-        var relevantDossierFiles = getRelevantFiles(dossierId);
+        var relevantDossierFiles = getRelevantFiles(dossierId, triggeredByUser);
 
         var relevantFiles = new ArrayList<FileEntity>();
         for (var fileId : fileIds) {
@@ -140,10 +127,6 @@ public class ReanalysisController implements ReanalysisResource {
 
             var dossierFile = dossierFileOptional.get();
 
-            if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
-                throw new ConflictException("Cannot analyse a deleted/approved file");
-            }
-
             relevantFiles.add(dossierFile);
 
         }
@@ -155,14 +138,14 @@ public class ReanalysisController implements ReanalysisResource {
 
         if (force) {
             filesToReanalyse.forEach(file -> {
-                fileStatusService.setStatusReprocess(dossierId, file.getId(), 2);
+                fileStatusService.setStatusReprocess(dossierId, file.getId(), 2, true);
             });
         } else {
             var enhancedAndConvertedFiles = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(
                     fileStatusService.getAllStatuses(), FileModel.class, new FileModelMapper()));
 
             enhancedAndConvertedFiles.forEach(file -> {
-                fileStatusService.setStatusReprocess(dossierId, file.getId(), 2);
+                fileStatusService.setStatusReprocess(dossierId, file.getId(), 2, true);
             });
         }
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionarySearchAndNotifyService.java

@@ -1,65 +0,0 @@
-package com.iqser.red.service.peristence.v1.server.service;
-
-import com.iqser.red.service.peristence.v1.server.client.SearchClient;
-import com.iqser.red.service.peristence.v1.server.controller.ReanalysisController;
-import com.iqser.red.service.search.v1.model.MatchedSection;
-import com.iqser.red.service.search.v1.model.SearchRequest;
-import lombok.Builder;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class DictionarySearchAndNotifyService {
-
-    private final ReanalysisController reanalysisClient;
-    private final SearchClient searchClient;
-
-    public void dictionaryEntriesUpdated(String dossierTemplateId, Collection<String> changedEntries) {
-
-        log.info("Detect Changes for dossierTemplateId: {} and entries: {}", dossierTemplateId, changedEntries);
-
-        Map<SearchKey, Set<Integer>> searchResultMap = new HashMap<>();
-
-        for (var entry : changedEntries) {
-
-
-            var searchResult = searchClient.getDossierStatus(SearchRequest.builder()
-                    .dossierTemplateIds(Collections.singletonList(dossierTemplateId)).page(1).pageSize(10_000)
-                    .returnSections(true)
-                    .queryString(entry)
-                    .build());
-
-            log.info("Got changes: {} for entry: {}", searchResult.getTotal(), entry);
-
-            searchResult.getMatchedDocuments().forEach(document -> {
-
-                var currentSections = searchResultMap.computeIfAbsent(SearchKey.builder().fileId(document.getFileId()).dossierId(document.getDossierId()).build(), (data) -> new HashSet<>());
-                currentSections.addAll(document.getMatchedSections().stream().map(MatchedSection::getSectionNumber).collect(Collectors.toList()));
-
-            });
-        }
-
-
-        log.info("Should reanalyse: {}", searchResultMap);
-        searchResultMap.forEach((searchKey, sectionNumbers) -> {
-            log.info("Reanalyse: {} / {} ", searchKey, sectionNumbers);
-            reanalysisClient.reanalyzeFile(searchKey.getDossierId(), searchKey.getFileId(), sectionNumbers);
-        });
-
-    }
-
-    @Data
-    @Builder
-    public static class SearchKey {
-        private String fileId;
-        private String dossierId;
-    }
-
-}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java

@@ -9,10 +9,8 @@ import org.springframework.stereotype.Service;
 @RequiredArgsConstructor
 public class ExcludeFromAnalysisService {
 
-
     private final FileStatusService fileStatusService;
 
-
     public void toggleExclusion(String dossierId, String fileId, boolean excluded) {
 
         var status = fileStatusService.getStatus(fileId);
@@ -31,4 +29,16 @@ public class ExcludeFromAnalysisService {
 
     }
 
+
+    public void toggleAutomaticAnalysis(String dossierId, String fileId, boolean excludedFromAutomaticAnalysis) {
+
+        // toggle status
+        fileStatusService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis);
+
+        if (!excludedFromAutomaticAnalysis) {
+            // if file has been re-enabled - process it
+            fileStatusService.setStatusFullReprocess(dossierId, fileId, 2);
+        }
+
+    }
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java

@@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java

@@ -31,7 +31,7 @@ public class FileStatusProcessingUpdateService {
 
             case SURROUNDING_TEXT:
                 fileStatusService.setStatusProcessed(analyzeResult.getFileId());
-                manualRedactionService.updateManualRedactions(fileId, analyzeResult.getManualRedactions());
+                manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions());
                 break;
             case REANALYSE:
             case FULL_ANALYSE:
@@ -47,6 +47,7 @@ public class FileStatusProcessingUpdateService {
                     indexingService.addToIndexingQueue(IndexMessageType.INSERT, dossier.getDossierTemplateId(), dossierId, fileId, 2);
                 }
 
+                manualRedactionService.updateProcessedDate(fileId, analyzeResult.getManualRedactions());
                 analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
         }
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java

@@ -12,13 +12,13 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.UserNotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.MessageType;
-import com.iqser.red.service.search.v1.model.IndexMessageType;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -128,12 +128,18 @@ public class FileStatusService {
     @Transactional
     public void setStatusReprocess(String dossierId, String fileId, int priority) {
 
-        setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet());
+        setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false);
+    }
+
+    @Transactional
+    public void setStatusReprocess(String dossierId, String fileId, int priority, boolean triggeredManually) {
+
+        setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually);
     }
 
 
     @Transactional
-    public void setStatusReprocess(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse) {
+    public void setStatusReprocess(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually) {
 
         log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
 
@@ -144,6 +150,11 @@ public class FileStatusService {
             return;
         }
 
+        if(fileStatus.isExcludedFromAutomaticAnalysis() && !triggeredManually){
+            log.debug("File {} is excluded from automatic analysis", fileStatus.getId());
+            return;
+        }
+
         fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS);
         addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse);
     }
@@ -241,6 +252,12 @@ public class FileStatusService {
     }
 
 
+    public void toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis) {
+
+        fileStatusPersistenceService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis);
+    }
+
+
     @Transactional
     protected void addToAnalysisQueue(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse) {
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionProviderService.java

@@ -5,7 +5,7 @@ import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper;
 import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
 import lombok.RequiredArgsConstructor;
 import org.springframework.stereotype.Service;

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java

@@ -13,21 +13,22 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.MessageType;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-
 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
+import javax.transaction.Transactional;
 import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
@@ -64,6 +65,7 @@ public class ManualRedactionService {
     private final HashFunction hashFunction = Hashing.murmur3_128();
 
 
+    @Transactional
     public ManualAddResponse addAddRedaction(String dossierId, String fileId, AddRedactionRequest addRedactionRequest) {
 
         dossierPersistenceService.getAndValidateDossier(dossierId);
@@ -83,6 +85,7 @@ public class ManualRedactionService {
 
         String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
         OffsetDateTime now = OffsetDateTime.now();
+
         addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
 
         Long commentId = null;
@@ -110,7 +113,6 @@ public class ManualRedactionService {
         return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build();
     }
 
-
     public ManualAddResponse addRemoveRedaction(String dossierId, String fileId,
                                                 RemoveRedactionRequest removeRedactionRequest) {
 
@@ -138,7 +140,7 @@ public class ManualRedactionService {
         }
 
         if (removeRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED)) {
-            fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+            reprocess(dossierId, fileId);
         }
 
         return ManualAddResponse.builder()
@@ -171,7 +173,7 @@ public class ManualRedactionService {
         analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
 
         if (forceRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED)) {
-            fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+            reprocess(dossierId, fileId);
         }
 
         return ManualAddResponse.builder()
@@ -228,7 +230,7 @@ public class ManualRedactionService {
             fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, now, true);
         } else {
             fileStatusPersistenceService.updateLastManualRedaction(fileId, now);
-            fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+            reprocess(dossierId, fileId);
         }
 
         analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
@@ -240,17 +242,6 @@ public class ManualRedactionService {
     }
 
 
-    public CommentEntity addComment(String fileId, String annotationId, CommentRequest commentRequest) {
-
-        var createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser());
-
-        fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
-        fileStatusPersistenceService.updateHasComments(fileId, true);
-
-        return createdComment;
-    }
-
-
     public ManualAddResponse addResizeRedaction(String dossierId, String fileId,
                                                 ResizeRedactionRequest resizeRedactionRequest) {
 
@@ -283,6 +274,16 @@ public class ManualRedactionService {
                 .build();
     }
 
+    public CommentEntity addComment(String fileId, String annotationId, CommentRequest commentRequest) {
+
+        var createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser());
+
+        fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
+        fileStatusPersistenceService.updateHasComments(fileId, true);
+
+        return createdComment;
+    }
+
 
     public ManualRedactionEntryEntity getAddRedaction(String fileId, String annotationId) {
 
@@ -408,7 +409,7 @@ public class ManualRedactionService {
             fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
         } else {
             fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
-            fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+            reprocess(dossierId, fileId);
         }
         analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
     }
@@ -456,7 +457,7 @@ public class ManualRedactionService {
                     .filter(entry -> entry.getId().equals(idRemoval.getId().getAnnotationId()))
                     .findFirst();
 
-            if (!redactionLogEntryOptional.isPresent()) {
+            if (redactionLogEntryOptional.isEmpty()) {
                 throw new NotFoundException("Annotation does not exist in redaction log.");
             }
 
@@ -482,16 +483,6 @@ public class ManualRedactionService {
     }
 
 
-    private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) {
-
-        if (redactionLogEntry.isDossierDictionaryEntry()) {
-            return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
-        } else {
-            return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
-        }
-    }
-
-
     public void updateForceRedactionStatus(String dossierId, String fileId, String annotationId,
                                            AnnotationStatus annotationStatus) {
 
@@ -503,6 +494,7 @@ public class ManualRedactionService {
     }
 
 
+    @Transactional
     public void updateLegalBasisChangeStatus(String dossierId, String fileId, String annotationId,
                                              AnnotationStatus annotationStatus) {
 
@@ -523,10 +515,10 @@ public class ManualRedactionService {
 
             // if it was previously approved, revert the delete
             if (imageRecategorization.getStatus() == AnnotationStatus.APPROVED) {
-                fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+                reprocess(dossierId, fileId);
             }
         } else if (annotationStatus.equals(AnnotationStatus.APPROVED)) {
-            fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+            reprocess(dossierId, fileId);
         }
 
         dossierPersistenceService.getAndValidateDossier(dossierId);
@@ -581,6 +573,40 @@ public class ManualRedactionService {
     }
 
 
+    @Transactional
+    public void updateSurroundingText(String fileId, ManualRedactions manualRedactions) {
+
+        // These are marked as processed once surrounding text is computed ( TBD if this is correct ? )
+        manualRedactions.getEntriesToAdd().forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
+        manualRedactions.getResizeRedactions().forEach(e -> resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
+    }
+
+
+    @Transactional
+    public void updateProcessedDate(String fileId, ManualRedactions manualRedactions) {
+
+        // These are marked as processed once analysis completes, not when they are set as approved
+        manualRedactions.getIdsToRemove().forEach(removeRedactionPersistenceService::markAsProcessed);
+        manualRedactions.getForceRedactions().forEach(forceRedactionPersistenceService::markAsProcessed);
+        manualRedactions.getImageRecategorization().forEach(recategorizationPersistenceService::markAsProcessed);
+
+    }
+
+
+    private void reprocess(String dossierId, String fileId) {
+        fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+    }
+
+
+    private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) {
+
+        if (redactionLogEntry.isDossierDictionaryEntry()) {
+            return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
+        } else {
+            return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
+        }
+    }
+
     private void handleAddToDictionary(String fileId, String annotationId, String typeId, String value,
                                        AnnotationStatus status, boolean addToDictionary, boolean addToDossierDictionary,
                                        boolean revert, String dossierId) {
@@ -615,7 +641,7 @@ public class ManualRedactionService {
                         .filter(entry -> entry.getId().equals(annotationId))
                         .findFirst();
 
-                if (!redactionLogEntryOptional.isPresent()) {
+                if (redactionLogEntryOptional.isEmpty()) {
                     throw new NotFoundException("Annotation does not exist in redaction log.");
                 }
 
@@ -638,7 +664,7 @@ public class ManualRedactionService {
             log.debug("Adding entries to {} for {} / {}", typeId, dossierId, fileId);
             dictionaryController.addEntries(typeId, List.of(value), false, false);
 
-            fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+            reprocess(dossierId, fileId);
         } catch (Exception e) {
             throw new BadRequestException(e.getMessage());
         }
@@ -650,7 +676,7 @@ public class ManualRedactionService {
         try {
             log.debug("Deleting entries to {} for {} / {}", typeId, dossierId, fileId);
             dictionaryController.deleteEntries(typeId, List.of(value));
-            fileStatusService.setStatusReprocess(dossierId, fileId, 2);
+            reprocess(dossierId, fileId);
         } catch (FeignException e) {
             throw new BadRequestException(e.getMessage());
         }
@@ -711,18 +737,6 @@ public class ManualRedactionService {
     }
 
 
-    public void updateManualRedactions(String fileId, ManualRedactions manualRedactions) {
-
-        manualRedactions.getEntriesToAdd().forEach(e -> {
-            addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter());
-        });
-
-        manualRedactions.getResizeRedactions().forEach(e -> {
-            resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter());
-        });
-    }
-
-
     private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) {
 
         fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING);

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java

@@ -67,7 +67,7 @@ public class AutomaticAnalysisScheduler {
                     if (next.isFullAnalysisRequired()) {
                         log.info("Queued file: {} for automatic full analysis! ", next.getFilename());
                         fileStatusService.setStatusFullReprocess(next.getDossierId(), next.getId(), 1);
-                    } else if(next.isReanalysisRequired()){
+                    } else if (next.isReanalysisRequired()) {
                         log.info("Queued file: {} for automatic reanalysis! ", next.getFilename());
                         fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1);
                     }
@@ -86,6 +86,7 @@ public class AutomaticAnalysisScheduler {
                         .stream()
                         .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED))
                         .filter(f -> !f.getWorkflowStatus().equals(WorkflowStatus.APPROVED))
+                        .filter(f -> !f.isExcludedFromAutomaticAnalysis())
                         .filter(f -> !isProcessing(f))
                         .filter(f -> f.getDossier().getSoftDeletedTime() == null && f.getDossier().getHardDeletedTime() == null && f.getDossier().getArchivedTime() == null)
                         .collect(Collectors.toList()), FileModel.class, new FileModelMapper()))

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/6-excluded-from-automatic-analysis-file-column.changelog.yaml

@@ -0,0 +1,24 @@
+databaseChangeLog:
+  - changeSet:
+      id: add-excluded-from-automatic-analysis-column
+      author: timo
+      changes:
+        - addColumn:
+            columns:
+              - column:
+                  name: excluded_from_automatic_analysis
+                  type: BOOLEAN
+                  defaultValue: false
+            tableName: file
+  - changeSet:
+      id: updated-excluded-from-automatic-analysis-column
+      author: timo
+      changes:
+        -  update:
+             columns:
+               -  column:
+                    name: excluded_from_automatic_analysis
+                    value: true
+             schemaName: public
+             tableName: file
+             where: workflow_status = 'APPROVED'

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -9,3 +9,5 @@ databaseChangeLog:
       file: db/changelog/4-archived-dossier.changelog.yaml
   - include:
       file: db/changelog/5-imported-annotation.changelog.yaml
+  - include:
+      file: db/changelog/6-excluded-from-automatic-analysis-file-column.changelog.yaml

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java

@@ -1,20 +1,20 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests;
 
-import static org.assertj.core.api.Assertions.assertThat;
-
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
 import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
 import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.MessageType;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import static org.assertj.core.api.Assertions.assertThat;
 
 public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
 
@@ -48,6 +48,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
 
         fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
+                .manualRedactions(new ManualRedactions())
                 .messageType(MessageType.FULL_ANALYSE)
                 .analysisVersion(100)
                 .fileId(file.getId())
@@ -97,6 +98,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
 
         fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
+                .manualRedactions(new ManualRedactions())
                 .messageType(MessageType.FULL_ANALYSE)
                 .analysisVersion(100)
                 .fileId(file.getId())
@@ -132,12 +134,13 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
 
 
     @Test
-    public void testFileReanlyseAfterToggleExclusion(){
+    public void testFileReanlyseAfterToggleExclusion() {
         var dossier = dossierTesterAndProvider.provideTestDossier();
 
         var file = fileTesterAndProvider.testAndProvideFile(dossier);
 
         fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
+                .manualRedactions(new ManualRedactions())
                 .messageType(MessageType.FULL_ANALYSE)
                 .analysisVersion(100)
                 .fileId(file.getId())

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java

@@ -291,8 +291,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .build());
         var loadedForceRedaction2 = manualRedactionClient.getForceRedaction(file.getId(), forceRedaction2.getAnnotationId());
         assertThat(loadedForceRedaction2.getStatus()).isEqualTo(AnnotationStatus.APPROVED);
-        assertThat(loadedForceRedaction2.getProcessedDate()).isNotNull();
+        assertThat(loadedForceRedaction2.getProcessedDate()).isNull();
-
 
         manualRedactionClient.updateForceRedactionStatus(dossier.getId(), file.getId(), forceRedaction2.getAnnotationId(), JSONPrimitive.of(AnnotationStatus.REQUESTED));
         loadedForceRedaction2 = manualRedactionClient.getForceRedaction(file.getId(), forceRedaction2.getAnnotationId());

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java

@@ -42,33 +42,28 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest {
         var file = fileTesterAndProvider.testAndProvideFile(dossier);
 
 
-        reanalysisClient.ocrDossier(dossier.getId());
+        reanalysisClient.ocrDossier(dossier.getId(),true);
         var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
         resetProcessingStatus(file);
 
-        reanalysisClient.ocrFile(dossier.getId(), file.getId(), true);
+        reanalysisClient.ocrFile(dossier.getId(), file.getId(),true, true);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
         resetProcessingStatus(file);
 
 
-        reanalysisClient.ocrFiles(dossier.getId(), Set.of(file.getId()));
+        reanalysisClient.ocrFiles(dossier.getId(),true, Set.of(file.getId()));
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
         resetProcessingStatus(file);
 
-        reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true);
+        reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true,true);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
         resetProcessingStatus(file);
 
-        reanalysisClient.reanalyzeFile(dossier.getId(), file.getId(), Sets.newHashSet());
+        reanalysisClient.reanalyzeDossier(dossier.getId(), true,true);
-        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
-        assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
-        resetProcessingStatus(file);
-
-        reanalysisClient.reanalyzeDossier(dossier.getId(), true);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
         resetProcessingStatus(file);