Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #82: RED-2513 If file is deleted, processState can not be updated

Browse Source 
Merge in RED/persistence-service from feature/RED-2513 to master

* commit 'd8303c5f4c9f1bb728175c01c3d4c16adec88702':
  RED-2513 If file is deleted, processState can not be updated
This commit is contained in:
Philipp Schramm 2021-10-25 09:24:35 +02:00
commit da22070042
2 changed files with 157 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -5,6 +5,7 @@ import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
@ -24,7 +25,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class FileStatusPersistenceService {
@ -35,6 +35,7 @@ public class FileStatusPersistenceService {
private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
private final DossierPersistenceService dossierService;
public void createStatus(String dossierId, String fileId, String filename, String uploader) {
FileEntity file = new FileEntity();
@ -58,6 +59,9 @@ public class FileStatusPersistenceService {
int analysisVersion) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setNumberOfPages(numberOfPages);
file.setProcessingStatus(ProcessingStatus.PROCESSED);
file.setDictionaryVersion(dictionaryVersion);
@ -66,7 +70,7 @@ public class FileStatusPersistenceService {
file.setAnalysisDuration(duration);
file.setDossierDictionaryVersion(dossierDictionaryVersion);
file.setAnalysisVersion(analysisVersion);
file.setNumberOfAnalyses(file.getNumberOfAnalyses()+1);
file.setNumberOfAnalyses(file.getNumberOfAnalyses() + 1);
file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setLastProcessed(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}, () -> {
@ -77,7 +81,12 @@ public class FileStatusPersistenceService {
@Transactional
public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates) {
public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages,
boolean hasSuggestions, boolean hasComments, boolean hasUpdates) {
if (isFileDeleted(fileId)) {
return;
}
fileRepository.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates);
}
@ -86,6 +95,9 @@ public class FileStatusPersistenceService {
public void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, boolean approval) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setWorkflowStatus(workflowStatus);
file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setApprovalDate(approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null);
@ -93,13 +105,16 @@ public class FileStatusPersistenceService {
throw new NotFoundException("Unknown file=" + fileId);
});
}
@Transactional
public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setProcessingStatus(processingStatus);
file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}, () -> {
@ -107,10 +122,14 @@ public class FileStatusPersistenceService {
});
}
@Transactional
public void setUpdateStatusIndexingSuccessful(String fileId) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setProcessingStatus(ProcessingStatus.PROCESSED);
file.setLastIndexed(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}, () -> {
@ -119,10 +138,14 @@ public class FileStatusPersistenceService {
}
@Transactional
public void updateLastOCRTime(String fileId, OffsetDateTime time) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setLastOCRTime(time);
}, () -> {
throw new NotFoundException("Unknown file=" + fileId);
@ -130,20 +153,28 @@ public class FileStatusPersistenceService {
}
@Transactional
public void updateHasComments(String fileId, boolean hasComments) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setHasAnnotationComments(hasComments);
}, () -> {
throw new NotFoundException("Unknown file=" + fileId);
});
}
@Transactional
public void updateLastManualRedaction(String fileId, OffsetDateTime date) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setLastManualRedaction(date);
}, () -> {
throw new NotFoundException("Unknown file=" + fileId);
@ -151,10 +182,15 @@ public class FileStatusPersistenceService {
}
@Transactional
public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date, boolean hasSuggestions) {
public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date,
boolean hasSuggestions) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setLastManualRedaction(date);
file.setHasSuggestions(hasSuggestions);
}, () -> {
@ -163,12 +199,16 @@ public class FileStatusPersistenceService {
}
@Transactional(value = Transactional.TxType.REQUIRES_NEW)
public void setFileAttributes(String dossierId, String fileId, Map<String, String> fileAttributes) {
fileAttributesRepository.deleteByFileId(fileId);
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
var fileAttributeEntities = convertFileAttributes(dossierId, file, fileAttributes);
file.setLastFileAttributeChange(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setFileAttributes(fileAttributeEntities);
@ -178,17 +218,20 @@ public class FileStatusPersistenceService {
}
@Transactional
public void setExcludedPages(String fileId, Set<Integer> excludedPages) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setLastManualRedaction(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setExcludedPages(excludedPages);
}, () -> {
throw new NotFoundException("Unknown file=" + fileId);
});
}
@ -200,13 +243,17 @@ public class FileStatusPersistenceService {
public List<FileEntity> getStatusesForDossier(String dossierId) {
return fileRepository.findByDossierId(dossierId);
}
public List<FileEntity> getAllStatuses() {
return fileRepository.findAll();
}
@Transactional
public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
@ -217,15 +264,16 @@ public class FileStatusPersistenceService {
throw new NotFoundException("Unknown file=" + fileId);
});
}
@Transactional
public void hardDelete(String fileId) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
file.setProcessingStatus(ProcessingStatus.DELETED);
file.setDeleted(file.getDeleted() == null ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : file.getDeleted());
file.setDeleted(file.getDeleted() == null ? OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS) : file.getDeleted());
file.setHardDeletedTime(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
fileAttributesRepository.deleteByFileId(fileId);
@ -235,6 +283,7 @@ public class FileStatusPersistenceService {
}
@Transactional
public void undelete(String fileId) {
@ -249,23 +298,31 @@ public class FileStatusPersistenceService {
});
}
@Transactional
public void setCurrentReviewer(String fileId, String currentReviewer, String lastReviewer) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setCurrentReviewer(currentReviewer);
file.setLastReviewer(lastReviewer);
file.setWorkflowStatus(currentReviewer == null ? WorkflowStatus.UNASSIGNED :WorkflowStatus.UNDER_REVIEW);
file.setWorkflowStatus(currentReviewer == null ? WorkflowStatus.UNASSIGNED : WorkflowStatus.UNDER_REVIEW);
}, () -> {
throw new NotFoundException("Unknown file=" + fileId);
});
}
@Transactional
public void toggleExclusion(String fileId, boolean excluded) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
return;
}
file.setExcluded(excluded);
file.setLastProcessed(null);
file.setLastReviewer(null);
@ -286,6 +343,7 @@ public class FileStatusPersistenceService {
}
@Transactional
public void overwriteFile(String fileId, String uploader, String filename) {
@ -318,7 +376,10 @@ public class FileStatusPersistenceService {
});
}
private List<FileAttributeEntity> convertFileAttributes(String dossierId, FileEntity file, Map<String, String> fileAttributesMap) {
private List<FileAttributeEntity> convertFileAttributes(String dossierId, FileEntity file,
Map<String, String> fileAttributesMap) {
var dossier = dossierService.getAndValidateDossier(dossierId);
List<FileAttributeConfigEntity> configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId());
@ -326,12 +387,35 @@ public class FileStatusPersistenceService {
var fa = new FileAttributeEntity();
fa.setFileAttributeId(new FileAttributeEntity.FileAttributeEntityId());
fa.setFile(file);
fa.setFileAttributeConfig(configuration.stream().filter(c -> c.getId().equals(entry.getKey())).findAny().orElseThrow(() ->
new BadRequestException("Invalid File Attribute Id")));
fa.setFileAttributeConfig(configuration.stream()
.filter(c -> c.getId().equals(entry.getKey()))
.findAny()
.orElseThrow(() -> new BadRequestException("Invalid File Attribute Id")));
fa.setValue(entry.getValue());
return fa;
}).collect(Collectors.toList());
}
private boolean isFileDeleted(FileEntity file) {
if (file == null || file.getDeleted() != null || file.getHardDeletedTime() != null || ProcessingStatus.DELETED.equals(file.getProcessingStatus())) {
return true;
}
return false;
}
private boolean isFileDeleted(String fileId) {
AtomicBoolean isFileDeleted = new AtomicBoolean(false);
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
isFileDeleted.set(isFileDeleted(file));
}, () -> {
throw new NotFoundException("Unknown file=" + fileId);
});
return isFileDeleted.get();
}
}

72
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java
View File

@ -1,17 +1,19 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient;
import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import static org.assertj.core.api.Assertions.assertThat;
public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
@ -27,8 +29,13 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
@Autowired
private FileClient fileClient;
@Autowired
private UploadClient uploadClient;
@Test
public void testFileProcessing() {
var dossier = dossierTesterAndProvider.provideTestDossier();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
@ -39,42 +46,85 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest {
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
.analysisVersion(100)
.fileId(file.getId())
.dossierId(dossier.getId()).build());
.dossierId(dossier.getId())
.build());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNASSIGNED);
assertThat(loadedFile.getAnalysisVersion()).isEqualTo(100);
fileProcessingClient.ocrFailed(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
fileProcessingClient.ocrSuccessful(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
assertThat(loadedFile.getLastOCRTime()).isNotNull();
fileProcessingClient.indexing(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.INDEXING);
assertThat(loadedFile.getLastIndexed()).isNull();
fileProcessingClient.indexingFailed(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
fileProcessingClient.indexingSuccessful(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNASSIGNED);
assertThat(loadedFile.getLastIndexed()).isNotNull();
}
@Test
public void testFileProcessingAfterDeletion() {
var dossier = dossierTesterAndProvider.provideTestDossier();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
assertThat(file.getLastOCRTime()).isNull();
fileProcessingClient.analysisFailed(dossier.getId(), file.getId());
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR);
fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder()
.analysisVersion(100)
.fileId(file.getId())
.dossierId(dossier.getId())
.build());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNASSIGNED);
assertThat(loadedFile.getAnalysisVersion()).isEqualTo(100);
// Delete file
uploadClient.deleteFile(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED);
assertThat(loadedFile.getDeleted()).isNotNull();
fileProcessingClient.ocrSuccessful(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED);
fileProcessingClient.indexing(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED);
fileProcessingClient.indexingFailed(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED);
fileProcessingClient.indexingSuccessful(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED);
}
}