RED-10088: Reanalyze all files in ERROR should ignore files from deleted dossiers

This commit is contained in:
Maverick Studer 2024-09-20 13:42:15 +02:00
parent 8b20e579a4
commit e78aa25d6c
11 changed files with 177 additions and 30 deletions

View File

@ -24,7 +24,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource;
@ -32,6 +31,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadRes
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalyzeFilesResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
@ -51,16 +51,16 @@ public class SupportController implements SupportResource {
@Override
public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
public ReanalyzeFilesResponse reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings);
return new ReanalyzeFilesResponse(reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings));
}
@Override
public void reanalyzeAllErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
public void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis) {
reanalysisService.reanalyzeAllErrorFiles(repeatStructureAnalysis);
reanalysisService.reanalyzeAllRelevantErrorFiles(repeatStructureAnalysis);
}

View File

@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalyzeFilesResponse;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Schema;
@ -53,6 +54,7 @@ public interface SupportResource {
String IMPORT = "/import";
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
@Operation(summary = "Reanalyze all files in dossier template", description = """
## Reanalyze Files Endpoint
@ -70,14 +72,14 @@ public interface SupportResource {
- **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated.
- **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null.
""")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings);
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
ReanalyzeFilesResponse reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings);
@PostMapping(value = ERROR_REANALYSIS_REST_PATH)
@Operation(summary = "Reanalyze all files in error state.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
void reanalyzeAllErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
void reanalyzeAllRelevantErrorFiles(@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
@PostMapping(value = ERROR_REANALYSIS_REST_PATH + DOSSIER_ID_PATH_VARIABLE + BULK_REST_PATH)

View File

@ -157,9 +157,9 @@ public class FileStatusService {
@Transactional
public List<FileModel> getAllErrorFiles() {
public List<FileModel> getAllRelevantErrorFiles() {
var fileEntities = new ArrayList<>(fileStatusPersistenceService.getAllErrorFiles());
var fileEntities = new ArrayList<>(fileStatusPersistenceService.getAllRelevantErrorFiles());
var convertedList = MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
}

View File

@ -83,6 +83,7 @@ public class ReanalysisRequiredStatusService {
}
fileStatus.setDossierArchived(dossier.getArchivedTime() != null);
fileStatus.setDossierDeleted(dossier.getHardDeletedTime() != null || dossier.getSoftDeletedTime() != null);
fileStatus.setDossierTemplateId(dossier.getDossierTemplateId());
fileStatus.setDossierStatusId(dossier.getDossierStatusId());

View File

@ -26,6 +26,7 @@ import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
@ -47,9 +48,9 @@ public class ReanalysisService {
private final Predicate<FileModel> errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR);
public void reanalyzeAllErrorFiles(boolean repeatStructureAnalysis) {
public void reanalyzeAllRelevantErrorFiles(boolean repeatStructureAnalysis) {
var errorFiles = fileStatusService.getAllErrorFiles();
var errorFiles = fileStatusService.getAllRelevantErrorFiles();
analyseFiles(repeatStructureAnalysis, errorFiles);
}
@ -267,7 +268,7 @@ public class ReanalysisService {
}
public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
public List<FileModel> reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds());
@ -276,25 +277,29 @@ public class ReanalysisService {
.filter(file -> isInList(file, reanalysisSettings))
.filter(reanalysisSettings.fileStatusFilter().asPredicate())
.peek(file -> log.info("Reanalyzing file {}", file.getId()))
.toList();
.collect(Collectors.toList());
validateFilesForReanalysis(files);
List<FileModel> rejectedFiles = filterInvalidFiles(files);
files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis()));
return rejectedFiles;
}
private void validateFilesForReanalysis(List<FileModel> files) {
private List<FileModel> filterInvalidFiles(List<FileModel> files) {
for (var file : files) {
if (file.isSoftOrHardDeleted()) {
throw new BadRequestException("Cannot reanalyse deleted file!");
}
if (file.getWorkflowStatus() == WorkflowStatus.APPROVED) {
throw new BadRequestException("Cannot reanalyse approved file!");
List<FileModel> rejectedFiles = new ArrayList<>();
Iterator<FileModel> iterator = files.iterator();
while (iterator.hasNext()) {
FileModel file = iterator.next();
if (file.isDossierDeleted() || file.getWorkflowStatus() == WorkflowStatus.APPROVED || file.isDossierArchived()) {
rejectedFiles.add(file);
iterator.remove();
}
}
return rejectedFiles;
}

View File

@ -427,9 +427,9 @@ public class FileStatusPersistenceService {
}
public List<FileEntity> getAllErrorFiles() {
public List<FileEntity> getAllRelevantErrorFiles() {
return fileRepository.getAllErrorFilesExcludeDeleted();
return fileRepository.getAllErrorFilesExcludeDeletedAndArchived();
}

View File

@ -308,8 +308,11 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
List<FileEntity> getSoftDeletedFiles(@Param("dossierIds") List<String> dossierIds);
@Query("select f from FileEntity f where f.processingStatus = 'ERROR' and f.deleted is null and f.hardDeletedTime is null ")
List<FileEntity> getAllErrorFilesExcludeDeleted();
@Query("select f from FileEntity f "
+ "inner join DossierEntity d on d.id = f.dossierId "
+ "where f.processingStatus = 'ERROR' and f.deleted is null and f.hardDeletedTime is null "
+ "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null ")
List<FileEntity> getAllErrorFilesExcludeDeletedAndArchived();
@Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f "

View File

@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired;
@ -103,4 +104,14 @@ public class DossierTesterAndProvider {
return provideTestDossier(testTemplate, dossierName);
}
public List<Dossier> provideTestDossiers(List<String> dossierNames) {
var testTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
return dossierNames.stream()
.map(dossierName -> provideTestDossier(testTemplate, dossierName))
.toList();
}
}

View File

@ -1,25 +1,33 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient;
import com.iqser.red.service.peristence.v1.server.integration.client.SupportClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalyzeFilesResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -35,6 +43,9 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
@Autowired
private DossierTesterAndProvider dossierTesterAndProvider;
@Autowired
private DossierClient dossierClient;
@Autowired
private SupportClient supportClient;
@ -44,6 +55,9 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
@Autowired
private FileProcessingClient fileProcessingClient;
@Autowired
private FileStatusManagementService fileStatusManagementService;
@Test
public void testReanalysis() {
@ -56,7 +70,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
setProcessingStatusToError(dossier, file);
supportClient.reanalyzeAllErrorFiles(true);
supportClient.reanalyzeAllRelevantErrorFiles(true);
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
@ -78,7 +92,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
assertThat(e.status()).isEqualTo(400);
setFiletoStatusHardDeleted(file);
setFileToStatusHardDeleted(file);
e = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier.getId(), List.of(file.getFileId()), true));
assertThat(e.status()).isEqualTo(400);
@ -138,7 +152,7 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
fileStatuses = supportClient.getFileStatusForDossierTemplate(file.getDossierTemplateId(), new FileStatusFilter(null, null, true, false));
assertThat(fileStatuses.size()).isEqualTo(1);
setFiletoStatusHardDeleted(file);
setFileToStatusHardDeleted(file);
fileStatuses = supportClient.getFileStatusForDossierTemplate(file.getDossierTemplateId(), new FileStatusFilter(null, null, true, false));
assertThat(fileStatuses.size()).isEqualTo(0);
@ -149,7 +163,99 @@ public class SupportControllerTest extends AbstractPersistenceServerServiceTest
}
private void setFiletoStatusHardDeleted(FileStatus file) {
@Test
public void testReanalysisWhenDossierDeleted() {
var dossiers = dossierTesterAndProvider.provideTestDossiers(List.of("dossier1", "dossier2"));
var dossier1 = dossiers.get(0);
var dossier2 = dossiers.get(1);
var file1 = fileTesterAndProvider.testAndProvideFile(dossier1, "file1");
var file2 = fileTesterAndProvider.testAndProvideFile(dossier2, "file2");
setProcessingStatusToError(dossier1, file1);
setProcessingStatusToError(dossier2, file2);
dossierClient.deleteDossier(dossier1.getId());
supportClient.reanalyzeAllRelevantErrorFiles(false);
FeignException feignException = assertThrows(FeignException.class, () -> fileClient.getFileStatus(dossier1.getId(), file1.getId()));
assertEquals(feignException.status(), 404);
FileStatus fileStatus = FileStatusMapper.toFileStatus(fileStatusManagementService.getFileStatus(file1.getId()));
assertThat(fileStatus.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSED);
var loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
setProcessingStatusToError(dossier2, file2);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true);
loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true));
assertEquals(feignException.status(), 404);
setProcessingStatusToError(dossier2, file2);
ReanalyzeFilesResponse reanalyzeFilesResponse = supportClient.reanalyzeFiles(dossier1.getDossierTemplateId(),
new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()),
Collections.emptySet(),
true,
new FileStatusFilter(null, null, true, true)));
loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
assertEquals(reanalyzeFilesResponse.getRejectedFiles().size(), 1);
assertEquals(reanalyzeFilesResponse.getRejectedFiles()
.get(0).getId(), file1.getId());
}
@Test
public void testReanalysisWhenDossierArchived() {
var dossiers = dossierTesterAndProvider.provideTestDossiers(List.of("dossier1", "dossier2"));
var dossier1 = dossiers.get(0);
var dossier2 = dossiers.get(1);
var file1 = fileTesterAndProvider.testAndProvideFile(dossier1, "file1");
var file2 = fileTesterAndProvider.testAndProvideFile(dossier2, "file2");
setProcessingStatusToError(dossier1, file1);
setProcessingStatusToError(dossier2, file2);
dossierClient.archiveDossiers(Collections.singleton(dossier1.getId()));
supportClient.reanalyzeAllRelevantErrorFiles(false);
FileStatus fileStatus = fileClient.getFileStatus(dossier1.getId(), file1.getId());
assertThat(fileStatus.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
var loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
setProcessingStatusToError(dossier2, file2);
supportClient.reanalyzeErrorFilesBulkForDossier(dossier2.getId(), List.of(), true);
loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
FeignException feignException = assertThrows(FeignException.class, () -> supportClient.reanalyzeErrorFilesBulkForDossier(dossier1.getId(), List.of(), true));
assertEquals(feignException.status(), 404);
setProcessingStatusToError(dossier2, file2);
ReanalyzeFilesResponse reanalyzeFilesResponse = supportClient.reanalyzeFiles(dossier1.getDossierTemplateId(),
new ReanalysisSettings(Set.of(dossier1.getId(), dossier2.getId()),
Collections.emptySet(),
true,
null));
loadedFile2 = fileClient.getFileStatus(dossier2.getId(), file2.getId());
assertThat(loadedFile2.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING);
assertEquals(reanalyzeFilesResponse.getRejectedFiles().size(), 1);
assertEquals(reanalyzeFilesResponse.getRejectedFiles()
.get(0).getId(), file1.getId());
}
private void setFileToStatusHardDeleted(FileStatus file) {
fileRepository.findById(file.getId())
.ifPresent((f) -> {

View File

@ -0,0 +1,18 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
@AllArgsConstructor
public class ReanalyzeFilesResponse {
private List<FileModel> rejectedFiles;
}

View File

@ -67,6 +67,7 @@ public class FileModel {
private Map<String, String> fileAttributes = new HashMap<>();
private String dossierId;
private boolean dossierArchived;
private boolean dossierDeleted;
private String dossierTemplateId;
private String dossierStatusId;
private OffsetDateTime redactionModificationDate;