From d8303c5f4c9f1bb728175c01c3d4c16adec88702 Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Mon, 25 Oct 2021 07:31:50 +0200 Subject: [PATCH] RED-2513 If file is deleted, processState can not be updated --- .../FileStatusPersistenceService.java | 108 ++++++++++++++++-- .../integration/tests/FileProcessingTest.java | 72 ++++++++++-- 2 files changed, 157 insertions(+), 23 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 16d618678..c99247f3d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -5,6 +5,7 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import javax.transaction.Transactional; @@ -24,7 +25,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import lombok.RequiredArgsConstructor; - @Service @RequiredArgsConstructor public class FileStatusPersistenceService { @@ -35,6 +35,7 @@ public class FileStatusPersistenceService { private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; private final DossierPersistenceService dossierService; + public void createStatus(String dossierId, String fileId, String filename, String uploader) { FileEntity file = new FileEntity(); @@ -58,6 +59,9 @@ public class FileStatusPersistenceService { int analysisVersion) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setNumberOfPages(numberOfPages); file.setProcessingStatus(ProcessingStatus.PROCESSED); file.setDictionaryVersion(dictionaryVersion); @@ -66,7 +70,7 @@ public class FileStatusPersistenceService { file.setAnalysisDuration(duration); file.setDossierDictionaryVersion(dossierDictionaryVersion); file.setAnalysisVersion(analysisVersion); - file.setNumberOfAnalyses(file.getNumberOfAnalyses()+1); + file.setNumberOfAnalyses(file.getNumberOfAnalyses() + 1); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setLastProcessed(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); }, () -> { @@ -77,7 +81,12 @@ public class FileStatusPersistenceService { @Transactional - public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates) { + public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, + boolean hasSuggestions, boolean hasComments, boolean hasUpdates) { + + if (isFileDeleted(fileId)) { + return; + } fileRepository.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates); } @@ -86,6 +95,9 @@ public class FileStatusPersistenceService { public void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, boolean approval) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setWorkflowStatus(workflowStatus); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setApprovalDate(approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null); @@ -93,13 +105,16 @@ public class FileStatusPersistenceService { throw new NotFoundException("Unknown file=" + fileId); }); - } @Transactional public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus) { + fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setProcessingStatus(processingStatus); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); }, () -> { @@ -107,10 +122,14 @@ public class FileStatusPersistenceService { }); } + @Transactional public void setUpdateStatusIndexingSuccessful(String fileId) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setProcessingStatus(ProcessingStatus.PROCESSED); file.setLastIndexed(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); }, () -> { @@ -119,10 +138,14 @@ public class FileStatusPersistenceService { } + @Transactional public void updateLastOCRTime(String fileId, OffsetDateTime time) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setLastOCRTime(time); }, () -> { throw new NotFoundException("Unknown file=" + fileId); @@ -130,20 +153,28 @@ public class FileStatusPersistenceService { } + @Transactional public void updateHasComments(String fileId, boolean hasComments) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setHasAnnotationComments(hasComments); }, () -> { throw new NotFoundException("Unknown file=" + fileId); }); } + @Transactional public void updateLastManualRedaction(String fileId, OffsetDateTime date) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setLastManualRedaction(date); }, () -> { throw new NotFoundException("Unknown file=" + fileId); @@ -151,10 +182,15 @@ public class FileStatusPersistenceService { } + @Transactional - public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date, boolean hasSuggestions) { + public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date, + boolean hasSuggestions) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setLastManualRedaction(date); file.setHasSuggestions(hasSuggestions); }, () -> { @@ -163,12 +199,16 @@ public class FileStatusPersistenceService { } + @Transactional(value = Transactional.TxType.REQUIRES_NEW) public void setFileAttributes(String dossierId, String fileId, Map fileAttributes) { fileAttributesRepository.deleteByFileId(fileId); fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } var fileAttributeEntities = convertFileAttributes(dossierId, file, fileAttributes); file.setLastFileAttributeChange(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setFileAttributes(fileAttributeEntities); @@ -178,17 +218,20 @@ public class FileStatusPersistenceService { } + @Transactional public void setExcludedPages(String fileId, Set excludedPages) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setLastManualRedaction(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setExcludedPages(excludedPages); }, () -> { throw new NotFoundException("Unknown file=" + fileId); }); - } @@ -200,13 +243,17 @@ public class FileStatusPersistenceService { public List getStatusesForDossier(String dossierId) { + return fileRepository.findByDossierId(dossierId); } + public List getAllStatuses() { + return fileRepository.findAll(); } + @Transactional public void softDelete(String fileId, OffsetDateTime softDeletedTime) { @@ -217,15 +264,16 @@ public class FileStatusPersistenceService { throw new NotFoundException("Unknown file=" + fileId); }); - } + @Transactional public void hardDelete(String fileId) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { file.setProcessingStatus(ProcessingStatus.DELETED); - file.setDeleted(file.getDeleted() == null ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : file.getDeleted()); + file.setDeleted(file.getDeleted() == null ? OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS) : file.getDeleted()); file.setHardDeletedTime(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); fileAttributesRepository.deleteByFileId(fileId); @@ -235,6 +283,7 @@ public class FileStatusPersistenceService { } + @Transactional public void undelete(String fileId) { @@ -249,23 +298,31 @@ public class FileStatusPersistenceService { }); } + @Transactional public void setCurrentReviewer(String fileId, String currentReviewer, String lastReviewer) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setCurrentReviewer(currentReviewer); file.setLastReviewer(lastReviewer); - file.setWorkflowStatus(currentReviewer == null ? WorkflowStatus.UNASSIGNED :WorkflowStatus.UNDER_REVIEW); + file.setWorkflowStatus(currentReviewer == null ? WorkflowStatus.UNASSIGNED : WorkflowStatus.UNDER_REVIEW); }, () -> { throw new NotFoundException("Unknown file=" + fileId); }); } + @Transactional public void toggleExclusion(String fileId, boolean excluded) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { + if (isFileDeleted(file)) { + return; + } file.setExcluded(excluded); file.setLastProcessed(null); file.setLastReviewer(null); @@ -286,6 +343,7 @@ public class FileStatusPersistenceService { } + @Transactional public void overwriteFile(String fileId, String uploader, String filename) { @@ -318,7 +376,10 @@ public class FileStatusPersistenceService { }); } - private List convertFileAttributes(String dossierId, FileEntity file, Map fileAttributesMap) { + + private List convertFileAttributes(String dossierId, FileEntity file, + Map fileAttributesMap) { + var dossier = dossierService.getAndValidateDossier(dossierId); List configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId()); @@ -326,12 +387,35 @@ public class FileStatusPersistenceService { var fa = new FileAttributeEntity(); fa.setFileAttributeId(new FileAttributeEntity.FileAttributeEntityId()); fa.setFile(file); - fa.setFileAttributeConfig(configuration.stream().filter(c -> c.getId().equals(entry.getKey())).findAny().orElseThrow(() -> - new BadRequestException("Invalid File Attribute Id"))); + fa.setFileAttributeConfig(configuration.stream() + .filter(c -> c.getId().equals(entry.getKey())) + .findAny() + .orElseThrow(() -> new BadRequestException("Invalid File Attribute Id"))); fa.setValue(entry.getValue()); return fa; }).collect(Collectors.toList()); } + private boolean isFileDeleted(FileEntity file) { + + if (file == null || file.getDeleted() != null || file.getHardDeletedTime() != null || ProcessingStatus.DELETED.equals(file.getProcessingStatus())) { + return true; + } + return false; + } + + + private boolean isFileDeleted(String fileId) { + + AtomicBoolean isFileDeleted = new AtomicBoolean(false); + fileRepository.findById(fileId).ifPresentOrElse((file) -> { + isFileDeleted.set(isFileDeleted(file)); + }, () -> { + throw new NotFoundException("Unknown file=" + fileId); + }); + + return isFileDeleted.get(); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java index 60b77b8a5..bd05a2b41 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java @@ -1,17 +1,19 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient; +import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import static org.assertj.core.api.Assertions.assertThat; public class FileProcessingTest extends AbstractPersistenceServerServiceTest { @@ -27,8 +29,13 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { @Autowired private FileClient fileClient; + @Autowired + private UploadClient uploadClient; + + @Test public void testFileProcessing() { + var dossier = dossierTesterAndProvider.provideTestDossier(); var file = fileTesterAndProvider.testAndProvideFile(dossier); @@ -39,42 +46,85 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); - fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder() .analysisVersion(100) .fileId(file.getId()) - .dossierId(dossier.getId()).build()); + .dossierId(dossier.getId()) + .build()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNASSIGNED); assertThat(loadedFile.getAnalysisVersion()).isEqualTo(100); - fileProcessingClient.ocrFailed(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); - fileProcessingClient.ocrSuccessful(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); assertThat(loadedFile.getLastOCRTime()).isNotNull(); - fileProcessingClient.indexing(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.INDEXING); assertThat(loadedFile.getLastIndexed()).isNull(); - fileProcessingClient.indexingFailed(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); - fileProcessingClient.indexingSuccessful(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNASSIGNED); assertThat(loadedFile.getLastIndexed()).isNotNull(); } + + + @Test + public void testFileProcessingAfterDeletion() { + + var dossier = dossierTesterAndProvider.provideTestDossier(); + + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + assertThat(file.getLastOCRTime()).isNull(); + + fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); + var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); + + fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder() + .analysisVersion(100) + .fileId(file.getId()) + .dossierId(dossier.getId()) + .build()); + loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNASSIGNED); + assertThat(loadedFile.getAnalysisVersion()).isEqualTo(100); + + // Delete file + uploadClient.deleteFile(dossier.getId(), file.getId()); + loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); + assertThat(loadedFile.getDeleted()).isNotNull(); + + fileProcessingClient.ocrSuccessful(dossier.getId(), file.getId()); + loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); + + fileProcessingClient.indexing(dossier.getId(), file.getId()); + loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); + + fileProcessingClient.indexingFailed(dossier.getId(), file.getId()); + loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); + + fileProcessingClient.indexingSuccessful(dossier.getId(), file.getId()); + loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); + + } + }