RED-10088: Reanalyze all files in ERROR should ignore files from deleted dossiers

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -24,7 +24,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
 import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
 import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource;
@@ -32,6 +31,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadRes
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalyzeFilesResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
@@ -51,16 +51,16 @@ public class SupportController implements SupportResource {
 
 
     @Override
-    public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
+    public ReanalyzeFilesResponse reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
 
-        reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings);
+        return new ReanalyzeFilesResponse(reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings));
     }
 
 
     @Override
-    public void reanalyzeAllErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
+    public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
 
-        reanalysisService.reanalyzeAllErrorFiles(repeatStructureAnalysis);
+        reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis);
     }
 
 

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalyzeFilesResponse;
 
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Schema;
@@ -53,6 +54,7 @@ public interface SupportResource {
     String IMPORT = "/import";
 
 
+    @ResponseStatus(value = HttpStatus.OK)
     @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
     @Operation(summary = "Reanalyze all files in dossier template", description = """
             ## Reanalyze Files Endpoint
@@ -70,14 +72,14 @@ public interface SupportResource {
             - **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated.
             - **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null.
             """)
-    @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings);
+    ReanalyzeFilesResponse reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings);
 
 
     @PostMapping(value = ERROR_REANALYSIS_REST_PATH)
     @Operation(summary = "Reanalyze all files in error state.", description = "None")
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
-    void reanalyzeAllErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
+    void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
 
 
     @PostMapping(value = ERROR_REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE + BULK_REST_PATH)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java

@@ -164,9 +164,9 @@ public class FileStatusService {
 
 
     @Transactional
-    public List<FileModel> getAllErrorFiles() {
+    public List<FileModel> getAllRelevantErrorFiles() {
 
-        var fileEntities = new ArrayList<>(fileStatusPersistenceService.getAllErrorFiles());
+        var fileEntities = new ArrayList<>(fileStatusPersistenceService.getAllRelevantErrorFiles());
         var convertedList = MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper());
         return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java

@@ -83,6 +83,7 @@ public class ReanalysisRequiredStatusService {
         }
 
         fileStatus.setDossierArchived(dossier.getArchivedTime() != null);
+        fileStatus.setDossierDeleted(dossier.getHardDeletedTime() != null || dossier.getSoftDeletedTime() != null);
         fileStatus.setDossierTemplateId(dossier.getDossierTemplateId());
         fileStatus.setDossierStatusId(dossier.getDossierStatusId());
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java

@@ -26,6 +26,7 @@ import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -47,9 +48,9 @@ public class ReanalysisService {
     private final Predicate<FileModel> errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR);
 
 
-    public void reanalyzeAllErrorFiles(boolean repeatStructureAnalysis) {
+    public void reanalyzeAllRelevantErrorFiles(boolean repeatStructureAnalysis) {
 
-        var errorFiles = fileStatusService.getAllErrorFiles();
+        var errorFiles = fileStatusService.getAllRelevantErrorFiles();
         analyseFiles(repeatStructureAnalysis, errorFiles);
     }
 
@@ -267,7 +268,7 @@ public class ReanalysisService {
     }
 
 
-    public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
+    public List<FileModel> reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
 
         requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds());
 
@@ -276,25 +277,29 @@ public class ReanalysisService {
                 .filter(file -> isInList(file, reanalysisSettings))
                 .filter(reanalysisSettings.fileStatusFilter().asPredicate())
                 .peek(file -> log.info("Reanalyzing file {}", file.getId()))
-                .toList();
+                .collect(Collectors.toList());
 
-        validateFilesForReanalysis(files);
+        List<FileModel> rejectedFiles = filterInvalidFiles(files);
 
         files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis()));
 
+        return rejectedFiles;
     }
 
 
-    private void validateFilesForReanalysis(List<FileModel> files) {
+    private List<FileModel> filterInvalidFiles(List<FileModel> files) {
 
-        for (var file : files) {
+        List<FileModel> rejectedFiles = new ArrayList<>();
-            if (file.isSoftOrHardDeleted()) {
+
-                throw new BadRequestException("Cannot reanalyse deleted file!");
+        Iterator<FileModel> iterator = files.iterator();
-            }
+        while (iterator.hasNext()) {
-            if (file.getWorkflowStatus() == WorkflowStatus.APPROVED) {
+            FileModel file = iterator.next();
-                throw new BadRequestException("Cannot reanalyse approved file!");
+            if (file.isDossierDeleted() || file.getWorkflowStatus() == WorkflowStatus.APPROVED || file.isDossierArchived()) {
+                rejectedFiles.add(file);
+                iterator.remove();
             }
         }
+        return rejectedFiles;
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -427,9 +427,9 @@ public class FileStatusPersistenceService {
     }
 
 
-    public List<FileEntity> getAllErrorFiles() {
+    public List<FileEntity> getAllRelevantErrorFiles() {
 
-        return fileRepository.getAllErrorFilesExcludeDeleted();
+        return fileRepository.getAllErrorFilesExcludeDeletedAndArchived();
     }
 
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -316,8 +316,11 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     List<FileEntity> getHardDeletedFiles();
 
 
-    @Query("select f from FileEntity f where f.processingStatus = 'ERROR' and f.deleted is null and f.hardDeletedTime is null ")
+    @Query("select f from FileEntity f "
-    List<FileEntity> getAllErrorFilesExcludeDeleted();
+           + "inner join DossierEntity d on d.id = f.dossierId "
+           + "where f.processingStatus = 'ERROR' and f.deleted is null and f.hardDeletedTime is null "
+           + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null ")
+    List<FileEntity> getAllErrorFilesExcludeDeletedAndArchived();
 
 
     @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f "

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java

@@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
+import java.util.List;
 import java.util.Set;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -103,4 +104,14 @@ public class DossierTesterAndProvider {
         return provideTestDossier(testTemplate, dossierName);
     }
 
+
+    public List<Dossier> provideTestDossiers(List<String> dossierNames) {
+
+        var testTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+
+        return dossierNames.stream()
+                .map(dossierName -> provideTestDossier(testTemplate, dossierName))
+                .toList();
+    }
+
 }

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SupportControllerTest.java

@@ -1,25 +1,33 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import java.time.Instant;
 import java.time.OffsetDateTime;
 import java.time.ZoneId;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.SupportClient;
 import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalyzeFilesResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
@@ -35,6 +43,9 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
     @Autowired
     private DossierTesterAndProvider dossierTesterAndProvider;
 
+    @Autowired
+    private DossierClient dossierClient;
+
     @Autowired
     private SupportClient supportClient;
 
@@ -44,6 +55,9 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
     @Autowired
     private FileProcessingClient fileProcessingClient;
 
+    @Autowired
+    private FileStatusManagementService fileStatusManagementService;
+
 
     @Test
     public void testReanalysis() {
@@ -56,7 +70,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
 
         setProcessingStatusToError(dossier, file);
 
-        supportClient.reanalyzeAllErrorFiles(true);
+        supportClient.reanalyzeAllRelevantErrorFiles(true);
         var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
 
@@ -78,7 +92,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
         e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
         assertThat(e.status()).isEqualTo(400);
 
-        setFiletoStatusHardDeleted(file);
+        setFileToStatusHardDeleted(file);
         e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
         assertThat(e.status()).isEqualTo(400);
 
@@ -138,7 +152,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
         fileStatuses = supportClient.getFileStatusForDossierTemplate(file.getDossierTemplateId(), new FileStatusFilter(null, null, true, false));
         assertThat(fileStatuses.size()).isEqualTo(1);
 
-        setFiletoStatusHardDeleted(file);
+        setFileToStatusHardDeleted(file);
 
         fileStatuses = supportClient.getFileStatusForDossierTemplate(file.getDossierTemplateId(), new FileStatusFilter(null, null, true, false));
         assertThat(fileStatuses.size()).isEqualTo(0);
@@ -149,7 +163,99 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
     }
 
 
-    private void setFiletoStatusHardDeleted(FileStatus file) {
+    @Test
+    public void testReanalysisWhenDossierDeleted() {
+
+        var dossiers = dossierTesterAndProvider.provideTestDossiers(List.of("dossier1", "dossier2"));
+        var dossier1 = dossiers.get(0);
+        var dossier2 = dossiers.get(1);
+        var file1 = fileTesterAndProvider.testAndProvideFile(dossier1, "file1");
+        var file2 = fileTesterAndProvider.testAndProvideFile(dossier2, "file2");
+
+        setProcessingStatusToError(dossier1, file1);
+        setProcessingStatusToError(dossier2, file2);
+
+        dossierClient.deleteDossier(dossier1.getId());
+
+        supportClient.reanalyzeAllRelevantErrorFiles(false);
+        FeignException feignException = assertThrows(FeignException.class, () -> fileClient.getFileStatus(dossier1.getId(), file1.getId()));
+        assertEquals(feignException.status(), 404);
+        FileStatus fileStatus = FileStatusMapper.toFileStatus(fileStatusManagementService.getFileStatus(file1.getId()));
+        assertThat(fileStatus.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSED);
+        var loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
+        assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
+
+        setProcessingStatusToError(dossier2, file2);
+
+        supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true);
+        loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
+        assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
+
+        feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true));
+        assertEquals(feignException.status(), 404);
+
+        setProcessingStatusToError(dossier2, file2);
+
+        ReanalyzeFilesResponse reanalyzeFilesResponse = supportClient.reanalyzeFiles(dossier1.getDossierTemplateId(),
+                                                                                     new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()),
+                                                                                                            Collections.emptySet(),
+                                                                                                            true,
+                                                                                                            new FileStatusFilter(null, null, true, true)));
+        loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
+        assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
+        assertEquals(reanalyzeFilesResponse.getRejectedFiles().size(), 1);
+        assertEquals(reanalyzeFilesResponse.getRejectedFiles()
+                             .get(0).getId(), file1.getId());
+
+    }
+
+
+    @Test
+    public void testReanalysisWhenDossierArchived() {
+
+        var dossiers = dossierTesterAndProvider.provideTestDossiers(List.of("dossier1", "dossier2"));
+        var dossier1 = dossiers.get(0);
+        var dossier2 = dossiers.get(1);
+        var file1 = fileTesterAndProvider.testAndProvideFile(dossier1, "file1");
+        var file2 = fileTesterAndProvider.testAndProvideFile(dossier2, "file2");
+
+        setProcessingStatusToError(dossier1, file1);
+        setProcessingStatusToError(dossier2, file2);
+
+        dossierClient.archiveDossiers(Collections.singleton(dossier1.getId()));
+
+        supportClient.reanalyzeAllRelevantErrorFiles(false);
+        FileStatus fileStatus = fileClient.getFileStatus(dossier1.getId(), file1.getId());
+        assertThat(fileStatus.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
+        var loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
+        assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
+
+        setProcessingStatusToError(dossier2, file2);
+
+        supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true);
+        loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
+        assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
+
+        FeignException feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true));
+        assertEquals(feignException.status(), 404);
+
+        setProcessingStatusToError(dossier2, file2);
+
+        ReanalyzeFilesResponse reanalyzeFilesResponse = supportClient.reanalyzeFiles(dossier1.getDossierTemplateId(),
+                                                                                     new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()),
+                                                                                                            Collections.emptySet(),
+                                                                                                            true,
+                                                                                                            null));
+        loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
+        assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
+        assertEquals(reanalyzeFilesResponse.getRejectedFiles().size(), 1);
+        assertEquals(reanalyzeFilesResponse.getRejectedFiles()
+                             .get(0).getId(), file1.getId());
+
+    }
+
+
+    private void setFileToStatusHardDeleted(FileStatus file) {
 
         fileRepository.findById(file.getId())
                 .ifPresent((f) -> {

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalyzeFilesResponse.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.persistence.service.v1.api.shared.model;
+
+import java.util.List;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+@AllArgsConstructor
+public class ReanalyzeFilesResponse {
+
+    private List<FileModel> rejectedFiles;
+
+}

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java

@@ -67,6 +67,7 @@ public class FileModel {
     private Map<String, String> fileAttributes = new HashMap<>();
     private String dossierId;
     private boolean dossierArchived;
+    private boolean dossierDeleted;
     private String dossierTemplateId;
     private String dossierStatusId;
     private OffsetDateTime redactionModificationDate;