From 2c06826c52665f28b2b6b7b932fc90b819e3064d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Wed, 20 Oct 2021 12:52:10 +0200 Subject: [PATCH] RED-2187: Splitted processing status from workflow status --- .../dossiertemplate/dossier/DossierStats.java | 8 ++- .../dossier/file/FileModel.java | 4 +- .../dossier/file/FileStatus.java | 5 -- .../dossier/file/ProcessingStatus.java | 5 ++ .../dossier/file/WorkflowStatus.java | 5 ++ .../v1/api/model/license/ReportData.java | 5 +- .../processor/entity/dossier/FileEntity.java | 8 ++- .../FileStatusPersistenceService.java | 60 +++++++------------ .../controller/FileAttributesController.java | 4 +- .../controller/FileStatusController.java | 17 +++--- .../controller/ReanalysisController.java | 11 ++-- .../server/service/DossierStatsService.java | 9 ++- .../service/ExcludeFromAnalysisService.java | 14 ++--- .../v1/server/service/FileService.java | 4 +- .../FileStatusProcessingUpdateService.java | 4 +- .../v1/server/service/FileStatusService.java | 37 ++++++------ .../v1/server/service/IndexingService.java | 4 +- .../server/service/LicenseReportService.java | 3 +- .../ReanalysisRequiredStatusService.java | 41 ++++++++----- .../scheduler/AutomaticAnalysisScheduler.java | 35 ++++++----- .../integration/tests/DossierStatsTest.java | 13 ++-- .../integration/tests/FileProcessingTest.java | 17 +++--- .../v1/server/integration/tests/FileTest.java | 15 ++--- .../integration/tests/ReanalysisTest.java | 48 ++++++--------- 24 files changed, 190 insertions(+), 186 deletions(-) delete mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileStatus.java create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/WorkflowStatus.java diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java index 31e7ed4b8..90ca2c83b 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java @@ -1,7 +1,5 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -9,6 +7,9 @@ import lombok.NoArgsConstructor; import java.util.Map; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + @Data @Builder @NoArgsConstructor @@ -22,5 +23,6 @@ public class DossierStats { private boolean hasSuggestionsFilePresent; // true if at least one file in the dossier has suggestions private boolean hasUpdatesFilePresent; //true if at least one file in the dossier has updates private boolean hasNoFlagsFilePresent; // true if at least one file in the dossier has none of the other flags - private Map fileCountPerStatus; + private Map fileCountPerProcessingStatus; + private Map fileCountPerWorkflowStatus; } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java index db8e730a9..83bc8aa8b 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java @@ -21,8 +21,8 @@ public class FileModel { @JsonProperty("fileId") private String id; private String filename; - private FileStatus status; - private FileStatus lastSuccessfulStatus; + private ProcessingStatus processingStatus; + private WorkflowStatus workflowStatus; private int numberOfPages; private OffsetDateTime added; private OffsetDateTime lastUpdated; diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileStatus.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileStatus.java deleted file mode 100644 index 2f91739a8..000000000 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileStatus.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file; - -public enum FileStatus { - UNPROCESSED, REPROCESS, PROCESSING, ERROR, DELETED, UNASSIGNED, UNDER_REVIEW, UNDER_APPROVAL, APPROVED, FULLREPROCESS, OCR_PROCESSING, INDEXING -} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java new file mode 100644 index 000000000..f70270a68 --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file; + +public enum ProcessingStatus { + UNPROCESSED, REPROCESS, PROCESSING, PROCESSED, ERROR, DELETED, FULLREPROCESS, OCR_PROCESSING, INDEXING +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/WorkflowStatus.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/WorkflowStatus.java new file mode 100644 index 000000000..9fa433590 --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/WorkflowStatus.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file; + +public enum WorkflowStatus { + UNASSIGNED, UNDER_REVIEW, UNDER_APPROVAL, APPROVED +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/ReportData.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/ReportData.java index 1aefd3a95..db697af60 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/ReportData.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/ReportData.java @@ -1,6 +1,5 @@ package com.iqser.red.service.persistence.service.v1.api.model.license; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -8,6 +7,8 @@ import lombok.NoArgsConstructor; import java.time.Instant; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + @Data @Builder @AllArgsConstructor @@ -21,7 +22,7 @@ public class ReportData { private Instant deletedDate; private int numberOfAnalyzedPages; private int numberOfOcrPages; - private FileStatus status; + private WorkflowStatus workflowStatus; private int analysisCount; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index faaa67166..4507935c9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -1,7 +1,9 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -30,11 +32,11 @@ public class FileEntity { @Column @Enumerated(EnumType.STRING) - private FileStatus status; + private ProcessingStatus processingStatus; @Column @Enumerated(EnumType.STRING) - private FileStatus lastSuccessfulStatus; + private WorkflowStatus workflowStatus; @Column private int numberOfPages; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index c58e05441..53f1c1a35 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -19,7 +19,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import lombok.RequiredArgsConstructor; @@ -40,7 +41,8 @@ public class FileStatusPersistenceService { file.setId(fileId); file.setDossier(dossierRepository.getOne(dossierId)); file.setFilename(filename); - file.setStatus(FileStatus.UNPROCESSED); + file.setProcessingStatus(ProcessingStatus.UNPROCESSED); + file.setWorkflowStatus(WorkflowStatus.UNASSIGNED); file.setNumberOfPages(0); file.setAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setUploader(uploader); @@ -51,14 +53,13 @@ public class FileStatusPersistenceService { @Transactional - public void updateStatusSuccessful(String fileId, int numberOfPages, FileStatus status, long dictionaryVersion, long rulesVersion, - long legalBasisVersion, long duration, long dossierDictionaryVersion, - int analysisVersion) { + public void updateWorkflowStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion, + long legalBasisVersion, long duration, long dossierDictionaryVersion, + int analysisVersion) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { file.setNumberOfPages(numberOfPages); - file.setStatus(status); - file.setLastSuccessfulStatus(status); + file.setProcessingStatus(ProcessingStatus.PROCESSED); file.setDictionaryVersion(dictionaryVersion); file.setRulesVersion(rulesVersion); file.setLegalBasisVersion(legalBasisVersion); @@ -91,11 +92,10 @@ public class FileStatusPersistenceService { @Transactional - public void updateStatusSuccessful(String fileId, FileStatus status, boolean approval) { + public void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, boolean approval) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { - file.setStatus(status); - file.setLastSuccessfulStatus(status); + file.setWorkflowStatus(workflowStatus); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setApprovalDate(approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null); }, () -> { @@ -107,27 +107,9 @@ public class FileStatusPersistenceService { @Transactional - public void updateStatus(String fileId, FileStatus status, String uploader) { - - if (status.equals(FileStatus.UNASSIGNED) || status.equals(FileStatus.UNPROCESSED)) { - throw new IllegalArgumentException("please use specific methods for these calls!"); - } - + public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { - file.setStatus(status); - file.setUploader(uploader); - file.setLastUploaded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - }, () -> { - throw new NotFoundException("Unknown file=" + fileId); - }); - - } - - @Transactional - public void updateStatus(String fileId, FileStatus status) { - fileRepository.findById(fileId).ifPresentOrElse((file) -> { - file.setStatus(status); + file.setProcessingStatus(processingStatus); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); }, () -> { throw new NotFoundException("Unknown file=" + fileId); @@ -135,10 +117,10 @@ public class FileStatusPersistenceService { } @Transactional - public void setUpdateStatusIndexingSuccessful(String fileId, FileStatus status) { + public void setUpdateStatusIndexingSuccessful(String fileId) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { - file.setStatus(status); + file.setProcessingStatus(ProcessingStatus.PROCESSED); file.setLastIndexed(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); }, () -> { throw new NotFoundException("Unknown file=" + fileId); @@ -238,7 +220,7 @@ public class FileStatusPersistenceService { public void softDelete(String fileId, OffsetDateTime softDeletedTime) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { - file.setStatus(FileStatus.DELETED); + file.setProcessingStatus(ProcessingStatus.DELETED); file.setDeleted(softDeletedTime); }, () -> { throw new NotFoundException("Unknown file=" + fileId); @@ -251,7 +233,7 @@ public class FileStatusPersistenceService { public void hardDelete(String fileId) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { - file.setStatus(FileStatus.DELETED); + file.setProcessingStatus(ProcessingStatus.DELETED); file.setDeleted(file.getDeleted() == null ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : file.getDeleted()); file.setHardDeletedTime(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); @@ -263,13 +245,13 @@ public class FileStatusPersistenceService { } @Transactional - public void undelete(String fileId, FileStatus statusBefore) { + public void undelete(String fileId) { fileRepository.findById(fileId).ifPresentOrElse((file) -> { if (file.getHardDeletedTime() != null) { throw new BadRequestException("Cannot undelete a hard-deleted dossier file!"); } - file.setStatus(statusBefore != null ? statusBefore : FileStatus.UNASSIGNED); + file.setProcessingStatus(ProcessingStatus.PROCESSED); file.setDeleted(null); }, () -> { throw new NotFoundException("Unknown file=" + fileId); @@ -282,7 +264,7 @@ public class FileStatusPersistenceService { fileRepository.findById(fileId).ifPresentOrElse((file) -> { file.setCurrentReviewer(currentReviewer); file.setLastReviewer(lastReviewer); - file.setStatus(currentReviewer == null ? FileStatus.UNASSIGNED :FileStatus.UNDER_REVIEW); + file.setWorkflowStatus(currentReviewer == null ? WorkflowStatus.UNASSIGNED :WorkflowStatus.UNDER_REVIEW); }, () -> { throw new NotFoundException("Unknown file=" + fileId); }); @@ -319,8 +301,8 @@ public class FileStatusPersistenceService { fileRepository.findById(fileId).ifPresentOrElse((file) -> { file.setFilename(filename); file.setUploader(uploader); - file.setStatus(FileStatus.FULLREPROCESS); - file.setLastSuccessfulStatus(null); + file.setProcessingStatus(ProcessingStatus.FULLREPROCESS); + file.setWorkflowStatus(WorkflowStatus.UNASSIGNED); file.setLastUploaded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setLastOCRTime(null); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java index f43a53dfb..716acbc05 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java @@ -12,7 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ImportCsvRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ImportCsvResponse; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.resources.FileAttributesResource; import com.iqser.red.service.persistence.service.v1.api.utils.SuppressFBWarnings; import com.opencsv.CSVParser; @@ -58,7 +58,7 @@ public class FileAttributesController implements FileAttributesResource { Map fileStatusByFilename = fileStatusService.getDossierStatus(dossierId) .stream() - .filter(f -> !f.getStatus().equals(FileStatus.DELETED)) + .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) .collect(Collectors.toMap(FileEntity::getFilename, Function.identity())); List> rows = getCsvRecords(importCsvRequest.getCsvFile(), generalConfiguration.getDelimiter()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java index d02d7184b..9707c9637 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java @@ -4,7 +4,8 @@ import com.iqser.red.service.peristence.v1.server.service.*; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.persistence.service.v1.api.resources.StatusResource; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.PathVariable; @@ -32,7 +33,7 @@ public class FileStatusController implements StatusResource { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getAllStatuses() .stream() - .filter(f -> !f.getStatus().equals(FileStatus.DELETED)) + .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) .collect(Collectors.toList()), FileModel.class, new FileModelMapper())); } @@ -42,7 +43,7 @@ public class FileStatusController implements StatusResource { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getDossierStatus(dossierId) .stream() - .filter(f -> !f.getStatus().equals(FileStatus.DELETED)) + .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) .collect(Collectors.toList()), FileModel.class, new FileModelMapper())); } @@ -51,7 +52,7 @@ public class FileStatusController implements StatusResource { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getDossierStatus(dossierId) .stream() - .filter(f -> f.getStatus().equals(FileStatus.DELETED) && f.getHardDeletedTime() == null) + .filter(f -> f.getProcessingStatus().equals(ProcessingStatus.DELETED) && f.getHardDeletedTime() == null) .collect(Collectors.toList()), FileModel.class, new FileModelMapper())); } @@ -71,7 +72,7 @@ public class FileStatusController implements StatusResource { @RequestParam(value = REVIEWER_ID_REQUEST_PARAM, required = false) String reviewerId) { fileStatusService.setCurrentReviewer(dossierId, fileId, reviewerId); - fileStatusService.setStatusSuccessful(fileId, reviewerId != null ? FileStatus.UNDER_REVIEW : FileStatus.UNASSIGNED); + fileStatusService.setStatusSuccessful(fileId, reviewerId != null ? WorkflowStatus.UNDER_REVIEW : WorkflowStatus.UNASSIGNED); analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } @@ -83,7 +84,7 @@ public class FileStatusController implements StatusResource { FileEntity fileStatus = fileStatusService.getStatus(fileId); String lastReviewer = fileStatus.getLastReviewer(); fileStatusService.setCurrentReviewer(dossierId, fileId, lastReviewer); - fileStatusService.setStatusSuccessful(fileId, lastReviewer != null ? FileStatus.UNDER_REVIEW : FileStatus.UNASSIGNED); + fileStatusService.setStatusSuccessful(fileId, lastReviewer != null ? WorkflowStatus.UNDER_REVIEW : WorkflowStatus.UNASSIGNED); analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } @@ -93,7 +94,7 @@ public class FileStatusController implements StatusResource { @RequestParam(value = APPROVER_ID_REQUEST_PARAM, required = false) String approverId) { fileStatusService.setCurrentReviewer(dossierId, fileId, approverId); - fileStatusService.setStatusSuccessful(fileId, approverId != null ? FileStatus.UNDER_APPROVAL : FileStatus.UNASSIGNED); + fileStatusService.setStatusSuccessful(fileId, approverId != null ? WorkflowStatus.UNDER_APPROVAL : WorkflowStatus.UNASSIGNED); analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } @@ -101,7 +102,7 @@ public class FileStatusController implements StatusResource { public void setStatusApproved(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - fileStatusService.setApprovalStatusSuccessful(fileId, FileStatus.APPROVED); + fileStatusService.setApprovalStatusSuccessful(fileId, WorkflowStatus.APPROVED); } public void toggleExclusion(@PathVariable(DOSSIER_ID_PARAM) String dossierId, diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index c5ca7d777..938efb663 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -10,7 +10,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.persistence.service.v1.api.resources.ReanalysisResource; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -103,7 +104,7 @@ public class ReanalysisController implements ReanalysisResource { fileStatusService.setStatusOcrProcessing(dossierId, fileId); } else { FileEntity dossierFile = fileStatusService.getStatus(fileId); - if (dossierFile.getStatus().equals(FileStatus.DELETED) || dossierFile.getStatus().equals(FileStatus.APPROVED)) { + if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { throw new ConflictException("Cannot analyse a deleted/approved file"); } if (dossierFile.getLastOCRTime() != null) { @@ -119,8 +120,8 @@ public class ReanalysisController implements ReanalysisResource { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); return fileStatusService.getDossierStatus(dossier.getId()) .stream() - .filter(fileStatus -> !fileStatus.getStatus().equals(FileStatus.DELETED)) - .filter(fileStatus -> !fileStatus.getStatus().equals(FileStatus.APPROVED)) + .filter(fileStatus -> !fileStatus.getProcessingStatus().equals(ProcessingStatus.DELETED)) + .filter(fileStatus -> !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) .collect(Collectors.toList()); } @@ -139,7 +140,7 @@ public class ReanalysisController implements ReanalysisResource { var dossierFile = dossierFileOptional.get(); - if (dossierFile.getStatus().equals(FileStatus.DELETED) || dossierFile.getStatus().equals(FileStatus.APPROVED)) { + if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { throw new ConflictException("Cannot analyse a deleted/approved file"); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java index 986d94b83..28b9e33cb 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java @@ -5,7 +5,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStats; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -55,8 +55,11 @@ public class DossierStatsService { .findAny().ifPresent( (v) -> dossierStats.setHasNoFlagsFilePresent(true) ); - Map fileCountPerStatus = files.stream().collect(Collectors.toMap(FileEntity::getStatus, e -> 1, Math::addExact)); - dossierStats.setFileCountPerStatus(fileCountPerStatus); + var fileCountPerProcessingStatus = files.stream().collect(Collectors.toMap(FileEntity::getProcessingStatus, e -> 1, Math::addExact)); + dossierStats.setFileCountPerProcessingStatus(fileCountPerProcessingStatus); + + var fileCountPerWorkflowStatus = files.stream().collect(Collectors.toMap(FileEntity::getWorkflowStatus, e -> 1, Math::addExact)); + dossierStats.setFileCountPerWorkflowStatus(fileCountPerWorkflowStatus); return dossierStats; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java index 8d014d6a5..f471dfea4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java @@ -1,18 +1,16 @@ package com.iqser.red.service.peristence.v1.server.service; -import com.google.common.collect.Sets; -import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; -import java.util.Set; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + +import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor public class ExcludeFromAnalysisService { - private final static Set VALID_STATES_FOR_EXCLUSION = Sets.newHashSet(FileStatus.UNASSIGNED, FileStatus.UNDER_REVIEW, FileStatus.UNDER_APPROVAL); private final FileStatusService fileStatusService; @@ -21,8 +19,8 @@ public class ExcludeFromAnalysisService { var status = fileStatusService.getStatus(fileId); - if (!VALID_STATES_FOR_EXCLUSION.contains(status.getStatus())) { - throw new BadRequestException("Files in status " + status.getStatus() + " cannot be excluded/included"); + if (WorkflowStatus.APPROVED.equals(status.getWorkflowStatus())) { + throw new BadRequestException("File in workflow status " + status.getWorkflowStatus() + " cannot be excluded/included"); } // toggle status diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java index 65f7948c2..4b020fc02 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java @@ -13,6 +13,8 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Annota import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -53,7 +55,7 @@ public class FileService { var existingStatus = retrieveStatus(fileId); if (existingStatus != null) { - if (existingStatus.getStatus().name().equals(AnnotationStatus.APPROVED.name())) { + if (existingStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { throw new ConflictException("File already exists in status APPROVED"); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 09733d155..178aad519 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -1,7 +1,6 @@ package com.iqser.red.service.peristence.v1.server.service; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -25,12 +24,11 @@ public class FileStatusProcessingUpdateService { public void analysisSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - var lastSuccessfulStatus = fileStatusService.getStatus(fileId).getLastSuccessfulStatus(); retryTemplate.execute(retryContext -> { log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext .getRetryCount()); - fileStatusService.setStatusSuccessful(dossierId, fileId, Objects.requireNonNullElse(lastSuccessfulStatus, FileStatus.UNASSIGNED), analyzeResult); + fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult); return null; }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index f1f89efbb..1ab9945fa 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -10,8 +10,9 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.UserNotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import lombok.RequiredArgsConstructor; @@ -68,25 +69,25 @@ public class FileStatusService { public void setStatusProcessing(String fileId) { - fileStatusPersistenceService.updateStatus(fileId, FileStatus.PROCESSING); + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING); } - public void setStatusSuccessful(String dossierId, String fileId, FileStatus status, AnalyzeResult analyzeResult) { + public void setStatusSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) { - fileStatusPersistenceService.updateStatusSuccessful(fileId, analyzeResult.getNumberOfPages(), status, analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion()); + fileStatusPersistenceService.updateWorkflowStatus(fileId, analyzeResult.getNumberOfPages(), analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion()); } - public void setStatusSuccessful(String fileId, FileStatus status) { + public void setStatusSuccessful(String fileId, WorkflowStatus workflowStatus) { - fileStatusPersistenceService.updateStatusSuccessful(fileId, status, false); + fileStatusPersistenceService.updateWorkflowStatus(fileId, workflowStatus, false); } - public void setApprovalStatusSuccessful(String fileId, FileStatus status) { + public void setApprovalStatusSuccessful(String fileId, WorkflowStatus workflowStatus) { - fileStatusPersistenceService.updateStatusSuccessful(fileId, status, true); + fileStatusPersistenceService.updateWorkflowStatus(fileId, workflowStatus, true); } @@ -117,7 +118,7 @@ public class FileStatusService { return; } - fileStatusPersistenceService.updateStatus(fileId, FileStatus.REPROCESS); + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS); addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse); } @@ -131,7 +132,7 @@ public class FileStatusService { return; } - fileStatusPersistenceService.updateStatus(fileId, FileStatus.OCR_PROCESSING); + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING); addToOcrQueue(dossierId, fileId, 2); } @@ -146,27 +147,26 @@ public class FileStatusService { return; } - fileStatusPersistenceService.updateStatus(fileId, FileStatus.FULLREPROCESS); + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS); addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet()); } public void setStatusIndexingSuccessful(String fileId) { - FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); - fileStatusPersistenceService.setUpdateStatusIndexingSuccessful(fileId, fileStatus.getLastSuccessfulStatus()); + fileStatusPersistenceService.setUpdateStatusIndexingSuccessful(fileId); } public void setStatusIndexing(String fileId) { - fileStatusPersistenceService.updateStatus(fileId, FileStatus.INDEXING); + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.INDEXING); } public void setStatusError(String fileId) { - fileStatusPersistenceService.updateStatus(fileId, FileStatus.ERROR); + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ERROR); } @@ -184,8 +184,7 @@ public class FileStatusService { public void setFileStatusUndeleted(String fileId) { - var status = fileStatusPersistenceService.getStatus(fileId); - fileStatusPersistenceService.undelete(fileId, status.getLastSuccessfulStatus()); + fileStatusPersistenceService.undelete(fileId); } @@ -232,8 +231,8 @@ public class FileStatusService { .fileAttributes(convert(fileAttributes)) .build(); - analyseRequest.setReanalyseOnlyIfPossible(!fileStatus.getStatus().equals(FileStatus.UNPROCESSED) && !fileStatus.getStatus() - .equals(FileStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed() + analyseRequest.setReanalyseOnlyIfPossible(!fileStatus.getProcessingStatus().equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus() + .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed() .isAfter(fileStatus.getLastFileAttributeChange()))); analyseRequest.setExcludedPages(fileStatus.getExcludedPages()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/IndexingService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/IndexingService.java index 3afeea0cc..9979de3af 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/IndexingService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/IndexingService.java @@ -6,7 +6,7 @@ import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfigu import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.search.v1.model.IndexMessage; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -48,7 +48,7 @@ public class IndexingService { for (Pair reindexDossierId : reindexDossierIds) { List fileStatuses = fileStatusService.getDossierStatus(reindexDossierId.getRight()); for (FileEntity fileStatus : fileStatuses) { - if (fileStatus.getStatus().equals(FileStatus.DELETED)) { + if (fileStatus.getProcessingStatus().equals(ProcessingStatus.DELETED)) { continue; } if (fileIds != null && !fileIds.isEmpty() && !fileIds.contains(fileStatus.getId())) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java index 1631ef55f..abd567702 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java @@ -86,13 +86,12 @@ public class LicenseReportService { reportData.setAddedDate(fileStatus.getAdded().toInstant()); reportData.setLastUpdatedDate(fileStatus.getLastUpdated() == null ? null : fileStatus.getLastUpdated().toInstant()); reportData.setDeletedDate(fileStatus.getDeleted() == null ? null : fileStatus.getDeleted().toInstant()); - reportData.setStatus(fileStatus.getStatus()); + reportData.setWorkflowStatus(fileStatus.getWorkflowStatus()); reportData.setNumberOfAnalyzedPages(fileStatus.getNumberOfPages()); reportData.setNumberOfOcrPages(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0); reportData.setAnalysisCount(fileStatus.getNumberOfAnalyses()); totalPagesAnalyzed.addAndGet(fileStatus.getNumberOfPages()); totalPagesOcr.addAndGet(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0); - reportData.setStatus(fileStatus.getStatus()); totalNumberOfAnalyses.addAndGet(fileStatus.getNumberOfAnalyses()); return reportData; })) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java index 16f0b349f..a622e2a35 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java @@ -6,6 +6,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -46,24 +48,31 @@ public class ReanalysisRequiredStatusService { Map> dossierTemplateVersionMap, Map dossierVersionMap, Map dossierMap) { - switch (fileStatus.getStatus()) { - case ERROR: - return true; - case UNASSIGNED: - case UNDER_REVIEW: - case UNDER_APPROVAL: - if (fileStatus.getLastProcessed() == null) { - return true; - } - if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) { - return true; - } else { - return requiresReanalysisBasedOnVersionDifference(fileStatus, dossierTemplateVersionMap, dossierVersionMap, dossierMap); - } - default: - return false; + + if(ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())){ + return true; } + if(ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) { + + switch (fileStatus.getWorkflowStatus()) { + case UNASSIGNED: + case UNDER_REVIEW: + case UNDER_APPROVAL: + if (fileStatus.getLastProcessed() == null) { + return true; + } + if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) { + return true; + } else { + return requiresReanalysisBasedOnVersionDifference(fileStatus, dossierTemplateVersionMap, dossierVersionMap, dossierMap); + } + default: + return false; + } + } + return false; + } private boolean requiresReanalysisBasedOnVersionDifference(FileModel fileStatus, diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java index 826ef8a82..eb3bcb2a4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java @@ -6,15 +6,19 @@ import com.iqser.red.service.peristence.v1.server.service.ReanalysisRequiredStat import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; + import org.springframework.amqp.core.AmqpAdmin; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; + import java.util.List; import java.util.stream.Collectors; @@ -25,15 +29,16 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ma @RequiredArgsConstructor public class AutomaticAnalysisScheduler { - @Value("${persistence-service.automaticAnalysis.pageFactor:500}") private int pageFactor; private final FileStatusService fileStatusService; private final ReanalysisRequiredStatusService reanalysisRequiredStatusService; private final AmqpAdmin amqpAdmin; + @PostConstruct protected void postConstruct() { + log.info("Automatic Analysis pageFactor: {}", pageFactor); } @@ -43,10 +48,8 @@ public class AutomaticAnalysisScheduler { var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); if (redactionQueueInfo != null) { - log.info("Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", - MessagingConfiguration.REDACTION_QUEUE, - redactionQueueInfo.getMessageCount(), - redactionQueueInfo.getConsumerCount()); + log.info("Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", MessagingConfiguration.REDACTION_QUEUE, redactionQueueInfo + .getMessageCount(), redactionQueueInfo.getConsumerCount()); // only 1 file in queue var consumerCount = redactionQueueInfo.getConsumerCount(); if (redactionQueueInfo.getMessageCount() <= consumerCount) { @@ -76,17 +79,21 @@ public class AutomaticAnalysisScheduler { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getAllStatuses() .stream() - .filter(f -> !f.getStatus().equals(FileStatus.DELETED)) - .filter(f -> !f.getStatus().equals(FileStatus.APPROVED)) + .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) + .filter(f -> !f.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) .filter(f -> !isProcessing(f)) - .collect(Collectors.toList()), FileModel.class, new FileModelMapper())).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList()); + .collect(Collectors.toList()), FileModel.class, new FileModelMapper())) + .stream() + .filter(FileModel::isAnalysisRequired) + .collect(Collectors.toList()); } + private boolean isProcessing(FileEntity file) { - return file.getStatus() == FileStatus.PROCESSING || - file.getStatus() == FileStatus.REPROCESS || - file.getStatus() == FileStatus.FULLREPROCESS || - file.getStatus() == FileStatus.INDEXING || - file.getStatus() == FileStatus.OCR_PROCESSING; + + return !file.getProcessingStatus().equals(ProcessingStatus.PROCESSED) && !file.getProcessingStatus() + .equals(ProcessingStatus.UNPROCESSED) && !file.getProcessingStatus() + .equals(ProcessingStatus.DELETED) && !file.getProcessingStatus().equals(ProcessingStatus.ERROR); } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java index 22320776c..af597611a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java @@ -8,7 +8,10 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStats; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + +import org.hibernate.jdbc.Work; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -78,7 +81,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest { //alter file 4 fileRepository.findById(file4.getId()).ifPresent((file)->{ file.setHasHints(true); - file.setStatus(FileStatus.APPROVED); + file.setWorkflowStatus(WorkflowStatus.APPROVED); fileRepository.save(file); }); } @@ -94,7 +97,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest { assertThat(dossierStats.isHasSuggestionsFilePresent()).isTrue(); assertThat(dossierStats.isHasUpdatesFilePresent()).isTrue(); assertThat(dossierStats.isHasNoFlagsFilePresent()).isFalse(); - assertThat(dossierStats.getFileCountPerStatus().get(FileStatus.PROCESSING)).isEqualTo(2); + assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.PROCESSING)).isEqualTo(2); } @Test @@ -114,7 +117,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest { assertThat(dossierStats2.isHasSuggestionsFilePresent()).isFalse(); assertThat(dossierStats2.isHasUpdatesFilePresent()).isFalse(); assertThat(dossierStats2.isHasNoFlagsFilePresent()).isTrue(); - assertThat(dossierStats2.getFileCountPerStatus().get(FileStatus.PROCESSING)).isEqualTo(1); - assertThat(dossierStats2.getFileCountPerStatus().get(FileStatus.APPROVED)).isEqualTo(1); + assertThat(dossierStats2.getFileCountPerWorkflowStatus().get(WorkflowStatus.UNASSIGNED)).isEqualTo(1); + assertThat(dossierStats2.getFileCountPerWorkflowStatus().get(WorkflowStatus.APPROVED)).isEqualTo(1); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java index 91124422d..60b77b8a5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java @@ -5,7 +5,8 @@ import com.iqser.red.service.peristence.v1.server.integration.client.FileProcess import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -36,7 +37,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.ERROR); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder() @@ -44,35 +45,35 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { .fileId(file.getId()) .dossierId(dossier.getId()).build()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNASSIGNED); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNASSIGNED); assertThat(loadedFile.getAnalysisVersion()).isEqualTo(100); fileProcessingClient.ocrFailed(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.ERROR); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); fileProcessingClient.ocrSuccessful(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); assertThat(loadedFile.getLastOCRTime()).isNotNull(); fileProcessingClient.indexing(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.INDEXING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.INDEXING); assertThat(loadedFile.getLastIndexed()).isNull(); fileProcessingClient.indexingFailed(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.ERROR); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); fileProcessingClient.indexingSuccessful(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNASSIGNED); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNASSIGNED); assertThat(loadedFile.getLastIndexed()).isNotNull(); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 07767d080..07fa7723b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -13,7 +13,8 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -57,7 +58,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(),"1"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getCurrentReviewer()).isEqualTo("1"); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNDER_REVIEW); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW); fileClient.excludePages(dossier.getId(), file.getId(), Sets.newHashSet(1)); @@ -73,27 +74,27 @@ public class FileTest extends AbstractPersistenceServerServiceTest { fileClient.setCurrentFileReviewer(dossier.getId(), file.getId(), "1"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getCurrentReviewer()).isEqualTo("1"); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNDER_REVIEW); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW); fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), "1"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNDER_APPROVAL); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_APPROVAL); fileClient.setStatusUnderReview(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNDER_REVIEW); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW); fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), "1"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNDER_APPROVAL); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_APPROVAL); fileClient.setStatusApproved(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.APPROVED); + assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.APPROVED); assertThat(uploadClient.getOriginal(dossier.getId(), file.getId())).isNotNull(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java index 8d40cd6b6..615b95c59 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java @@ -7,7 +7,9 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + import org.assertj.core.util.Sets; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -42,61 +44,49 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest { reanalysisClient.ocrDossier(dossier.getId()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.OCR_PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); + resetProcessingStatus(file); - - resetStatus(file); - loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNASSIGNED); - - - resetStatus(file); reanalysisClient.ocrFile(dossier.getId(), file.getId(), true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.OCR_PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); + resetProcessingStatus(file); - resetStatus(file); reanalysisClient.ocrFiles(dossier.getId(), Set.of(file.getId())); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.OCR_PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); + resetProcessingStatus(file); - - resetStatus(file); reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); + resetProcessingStatus(file); - - resetStatus(file); reanalysisClient.reanalyzeFile(dossier.getId(), file.getId(), Sets.newHashSet()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); + resetProcessingStatus(file); - - resetStatus(file); reanalysisClient.reanalyzeDossier(dossier.getId(), true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.PROCESSING); - - - resetStatus(file); - reanalysisClient.reindex(dossier.getId(), Set.of(file.getId())); - fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); + resetProcessingStatus(file); } - private void resetStatus(FileModel file) { + private void resetProcessingStatus(FileModel file) { fileRepository.findById(file.getId()).ifPresent(savedFile -> { - savedFile.setStatus(FileStatus.UNASSIGNED); + savedFile.setProcessingStatus(ProcessingStatus.UNPROCESSED); fileRepository.save(savedFile); }); var loadedFile = fileClient.getFileStatus(file.getDossierId(), file.getId()); - assertThat(loadedFile.getStatus()).isEqualTo(FileStatus.UNASSIGNED); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.UNPROCESSED); } + }