Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Reanalyze by hand should also do full reprocess if required

Browse Source
This commit is contained in:
Timo Bejan 2023-07-28 10:18:25 +02:00
parent c3d0a9c2ed
commit 92bf9fd224
4 changed files with 24 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -81,6 +81,7 @@ public class FileStatusService {
var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(fileManagementServiceSettings.getMaxErrorRetries());
var convertedList = MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper());
log.info("Initial list of files for scheduler contains: {} files.", fileEntities.size());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList());
}

17
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java
View File

@ -70,14 +70,21 @@ public class ReanalysisService {
if (force) {
filesToReanalyse.forEach(file -> {
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
if (file.isFullAnalysisRequired()) {
fileStatusService.setStatusFullReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, false);
} else {
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
}
});
} else {
filesToReanalyse.stream()
.filter(FileModel::isReanalysisRequired)
.forEach(file -> fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true));
filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> {
if (file.isFullAnalysisRequired()) {
fileStatusService.setStatusFullReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, false);
} else {
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
}
});
}
}

13
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java
View File

@ -45,7 +45,7 @@ public class AutomaticAnalysisJob implements Job {
tenantProvider.getTenants().forEach(tenant -> {
if(!TenantUtils.isTenantReadyForPersistence(tenant)){
if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
return;
}
@ -53,7 +53,8 @@ public class AutomaticAnalysisJob implements Job {
var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
if (redactionQueueInfo != null) {
log.info("Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
log.info("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
tenant.getTenantId(),
MessagingConfiguration.REDACTION_QUEUE,
redactionQueueInfo.getMessageCount(),
redactionQueueInfo.getConsumerCount());
@ -66,7 +67,7 @@ public class AutomaticAnalysisJob implements Job {
allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated));
var allStatusesIterator = allStatuses.iterator();
log.info("Files that require reanalysis: {}", allStatuses.size());
log.info("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size());
var queuedFiles = 0;
@ -75,10 +76,10 @@ public class AutomaticAnalysisJob implements Job {
// in case the file doesn't have numberOfPages set, we assume an average.
if (next.isFullAnalysisRequired()) {
log.info("Queued file: {} for automatic full analysis! ", next.getFilename());
log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), next.getFilename());
fileStatusService.setStatusFullReprocess(next.getDossierId(), next.getId(), false, false);
} else if (next.isReanalysisRequired()) {
log.info("Queued file: {} for automatic reanalysis! ", next.getFilename());
log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), next.getFilename());
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false);
}
@ -87,7 +88,7 @@ public class AutomaticAnalysisJob implements Job {
}
} else {
log.info("Failed to obtain queue info for queue: {}", MessagingConfiguration.REDACTION_QUEUE);
log.info("[Tenant:{}] Failed to obtain queue info for queue: {}", TenantContext.getTenantId(), MessagingConfiguration.REDACTION_QUEUE);
}
});

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -183,7 +183,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate, OffsetDateTime lastUpdated);
@Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )" + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries")
@Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false "
+ " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )"
+ " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null "
+ " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries")
List<FileEntity> getAllRelevantStatusesForReanalysisScheduler(int maxRetries);