diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 30f299e4c..7e78c6cfa 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -96,6 +96,23 @@ public class SaasMigrationService implements TenantSyncService { for (var file : files) { + var dossier = dossierService.getDossierById(file.getDossierId()); + + if (dossier.getHardDeletedTime() != null) { + if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) { + saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED); + continue; + } else { + fileStatusPersistenceService.hardDelete(file.getFileId(), dossier.getHardDeletedTime()); + saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED); + continue; + } + } + if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) { + saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED); + continue; + } + if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) { log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus()); continue; @@ -113,11 +130,31 @@ public class SaasMigrationService implements TenantSyncService { } log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId()); + if (numberOfFiles == 0) { + finalizeMigration(); + } } public void startMigrationForFile(String dossierId, String fileId) { + var dossier = dossierService.getDossierById(dossierId); + + if (dossier.getHardDeletedTime() != null) { + if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) { + saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); + return; + } else { + fileStatusPersistenceService.hardDelete(fileId, dossier.getHardDeletedTime()); + saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); + return; + } + } + if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) { + saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); + return; + } + log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId); saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId); var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, false); @@ -125,15 +162,6 @@ public class SaasMigrationService implements TenantSyncService { } - public void requeueErrorFiles() { - - automaticAnalysisJob.stopForTenant(TenantContext.getTenantId()); - saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR) - .forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId())); - - } - - public void handleLayoutParsingFinished(String dossierId, String fileId) { if (!layoutParsingFilesExist(dossierId, fileId)) { @@ -154,14 +182,14 @@ public class SaasMigrationService implements TenantSyncService { String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId(); rabbitTemplate.convertAndSend(MIGRATION_QUEUE, - MigrationRequest.builder() - .dossierTemplateId(dossierTemplateId) - .dossierId(dossierId) - .fileId(fileId) - .fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) - .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted())) - .entitiesWithComments(commentService.getCommentCounts(fileId).keySet()) - .build()); + MigrationRequest.builder() + .dossierTemplateId(dossierTemplateId) + .dossierId(dossierId) + .fileId(fileId) + .fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED)) + .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted())) + .entitiesWithComments(commentService.getCommentCounts(fileId).keySet()) + .build()); } catch (Exception e) { log.error("Queuing of entityLog migration failed with {}", e.getMessage()); saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage())); @@ -173,9 +201,9 @@ public class SaasMigrationService implements TenantSyncService { private boolean layoutParsingFilesExist(String dossierId, String fileId) { return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) // - && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) // - && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) // - && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)); + && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) // + && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) // + && storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION)); } @@ -214,6 +242,13 @@ public class SaasMigrationService implements TenantSyncService { } } + public void requeueErrorFiles() { + + automaticAnalysisJob.stopForTenant(TenantContext.getTenantId()); + saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR) + .forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId())); + + } private void migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(String dossierId, String fileId) { @@ -282,10 +317,10 @@ public class SaasMigrationService implements TenantSyncService { updateAnnotationIds(fileId, idMapping); } catch (Exception e) { String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s", - TenantContext.getTenantId(), - dossierId, - fileId, - e.getMessage()); + TenantContext.getTenantId(), + dossierId, + fileId, + e.getMessage()); saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message); log.error(message); throw e; @@ -296,7 +331,7 @@ public class SaasMigrationService implements TenantSyncService { private void finalizeMigration() { if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) { -// automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); +// automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build()); log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId()); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 4ca594587..1bfe44a87 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -403,7 +403,10 @@ public class FileStatusPersistenceService { public List getStatusesForDossierTemplate(String dossierTemplateId) { return dossierService.findAllDossiersForDossierTemplateId(dossierTemplateId) - .stream().map(dossier -> fileRepository.findByDossierId(dossier.getId())).flatMap(List::stream).toList(); + .stream() + .map(dossier -> fileRepository.findByDossierId(dossier.getId())) + .flatMap(List::stream) + .toList(); } @@ -444,6 +447,12 @@ public class FileStatusPersistenceService { public void hardDelete(String fileId) { var now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + hardDelete(fileId, now); + } + + + @Transactional + public void hardDelete(String fileId, OffsetDateTime now) { fileRepository.hardDeleteFiles(List.of(fileId), ProcessingStatus.PROCESSED, now); fileAttributesRepository.deleteByFileId(fileId); @@ -453,7 +462,7 @@ public class FileStatusPersistenceService { @Transactional public void softDelete(String fileId, OffsetDateTime softDeletedTime) { - fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime); + fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime); }