From 92bf9fd224273cc29eaefa5ecf138780cc47540f Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 28 Jul 2023 10:18:25 +0200 Subject: [PATCH] Reanalyze by hand should also do full reprocess if required --- .../v1/processor/service/FileStatusService.java | 1 + .../v1/processor/service/ReanalysisService.java | 17 ++++++++++++----- .../service/job/AutomaticAnalysisJob.java | 13 +++++++------ .../persistence/repository/FileRepository.java | 5 ++++- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index b6a93ffc7..716246459 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -81,6 +81,7 @@ public class FileStatusService { var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(fileManagementServiceSettings.getMaxErrorRetries()); var convertedList = MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper()); + log.info("Initial list of files for scheduler contains: {} files.", fileEntities.size()); return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList()); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index 98ac5cd1d..7677dec2a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -70,14 +70,21 @@ public class ReanalysisService { if (force) { filesToReanalyse.forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); + if (file.isFullAnalysisRequired()) { + fileStatusService.setStatusFullReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, false); + } else { + fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); + } }); } else { - filesToReanalyse.stream() - .filter(FileModel::isReanalysisRequired) - .forEach(file -> fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true)); + filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> { + if (file.isFullAnalysisRequired()) { + fileStatusService.setStatusFullReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, false); + } else { + fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); + } + }); } - } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index f782fb779..2641a9b06 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -45,7 +45,7 @@ public class AutomaticAnalysisJob implements Job { tenantProvider.getTenants().forEach(tenant -> { - if(!TenantUtils.isTenantReadyForPersistence(tenant)){ + if (!TenantUtils.isTenantReadyForPersistence(tenant)) { return; } @@ -53,7 +53,8 @@ public class AutomaticAnalysisJob implements Job { var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); if (redactionQueueInfo != null) { - log.info("Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", + log.info("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", + tenant.getTenantId(), MessagingConfiguration.REDACTION_QUEUE, redactionQueueInfo.getMessageCount(), redactionQueueInfo.getConsumerCount()); @@ -66,7 +67,7 @@ public class AutomaticAnalysisJob implements Job { allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated)); var allStatusesIterator = allStatuses.iterator(); - log.info("Files that require reanalysis: {}", allStatuses.size()); + log.info("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size()); var queuedFiles = 0; @@ -75,10 +76,10 @@ public class AutomaticAnalysisJob implements Job { // in case the file doesn't have numberOfPages set, we assume an average. if (next.isFullAnalysisRequired()) { - log.info("Queued file: {} for automatic full analysis! ", next.getFilename()); + log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), next.getFilename()); fileStatusService.setStatusFullReprocess(next.getDossierId(), next.getId(), false, false); } else if (next.isReanalysisRequired()) { - log.info("Queued file: {} for automatic reanalysis! ", next.getFilename()); + log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), next.getFilename()); fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false); } @@ -87,7 +88,7 @@ public class AutomaticAnalysisJob implements Job { } } else { - log.info("Failed to obtain queue info for queue: {}", MessagingConfiguration.REDACTION_QUEUE); + log.info("[Tenant:{}] Failed to obtain queue info for queue: {}", TenantContext.getTenantId(), MessagingConfiguration.REDACTION_QUEUE); } }); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 191ff3845..20b6d6547 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -183,7 +183,10 @@ public interface FileRepository extends JpaRepository { void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate, OffsetDateTime lastUpdated); - @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )" + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries") + @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )" + + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries") List getAllRelevantStatusesForReanalysisScheduler(int maxRetries);