Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

migration-fp-hotfix #617

Merged
kilian.schuettler1 merged 3 commits from hotfix-migration-fp into master 2024-07-26 14:54:01 +02:00
Conversation 7 Commits 3 Files Changed 2 +71 -27
2 changed files with 71 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -96,6 +96,23 @@ public class SaasMigrationService implements TenantSyncService {
for (var file : files) {
var dossier = dossierService.getDossierById(file.getDossierId());
if (dossier.getHardDeletedTime() != null) {
if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
continue;
} else {
fileStatusPersistenceService.hardDelete(file.getFileId(), dossier.getHardDeletedTime());
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
continue;
}
}
if (fileStatusPersistenceService.getStatus(file.getFileId()).getHardDeletedTime() != null) {
saasMigrationStatusPersistenceService.updateStatus(file.getFileId(), SaasMigrationStatus.FINISHED);
continue;
}
if (!file.getStatus().equals(SaasMigrationStatus.MIGRATION_REQUIRED)) {
log.info("Skipping {} for tenant {} since migration status is {}", file.getFileId(), TenantContext.getTenantId(), file.getStatus());
continue;
@ -113,11 +130,31 @@ public class SaasMigrationService implements TenantSyncService {
}
log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId());
if (numberOfFiles == 0) {
finalizeMigration();
}
}
public void startMigrationForFile(String dossierId, String fileId) {
var dossier = dossierService.getDossierById(dossierId);
if (dossier.getHardDeletedTime() != null) {
if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
return;
} else {
fileStatusPersistenceService.hardDelete(fileId, dossier.getHardDeletedTime());
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
return;
}
}
if (fileStatusPersistenceService.getStatus(fileId).getHardDeletedTime() != null) {
saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
return;
}
log.info("Starting Migration for dossierId {} and fileId {}", dossierId, fileId);
saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossierId, fileId);
var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, false);
@ -125,15 +162,6 @@ public class SaasMigrationService implements TenantSyncService {
}
public void requeueErrorFiles() {
automaticAnalysisJob.stopForTenant(TenantContext.getTenantId());
saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR)
.forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId()));
}
public void handleLayoutParsingFinished(String dossierId, String fileId) {
if (!layoutParsingFilesExist(dossierId, fileId)) {
@ -154,14 +182,14 @@ public class SaasMigrationService implements TenantSyncService {
String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId();
rabbitTemplate.convertAndSend(MIGRATION_QUEUE,
MigrationRequest.builder()
.dossierTemplateId(dossierTemplateId)
.dossierId(dossierId)
.fileId(fileId)
.fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED))
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
.entitiesWithComments(commentService.getCommentCounts(fileId).keySet())
.build());
MigrationRequest.builder()
.dossierTemplateId(dossierTemplateId)
.dossierId(dossierId)
.fileId(fileId)
.fileIsApproved(fileStatusPersistenceService.getStatus(fileId).getWorkflowStatus().equals(WorkflowStatus.APPROVED))
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.allWithoutDeleted()))
.entitiesWithComments(commentService.getCommentCounts(fileId).keySet())
.build());
} catch (Exception e) {
log.error("Queuing of entityLog migration failed with {}", e.getMessage());
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage()));
@ -173,9 +201,9 @@ public class SaasMigrationService implements TenantSyncService {
private boolean layoutParsingFilesExist(String dossierId, String fileId) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) //
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) //
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) //
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION));
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) //
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) //
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION));
}
@ -214,6 +242,13 @@ public class SaasMigrationService implements TenantSyncService {
}
}
public void requeueErrorFiles() {
automaticAnalysisJob.stopForTenant(TenantContext.getTenantId());
saasMigrationStatusPersistenceService.findAllByStatus(SaasMigrationStatus.ERROR)
.forEach(migrationStatus -> startMigrationForFile(migrationStatus.getDossierId(), migrationStatus.getFileId()));
}
private void migrateAnnotationIdsAndAddManualAddRedactionsAndDeleteSectionGrid(String dossierId, String fileId) {
@ -282,10 +317,10 @@ public class SaasMigrationService implements TenantSyncService {
updateAnnotationIds(fileId, idMapping);
} catch (Exception e) {
String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s",
TenantContext.getTenantId(),
dossierId,
fileId,
e.getMessage());
TenantContext.getTenantId(),
dossierId,
fileId,
e.getMessage());
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message);
log.error(message);
throw e;
@ -296,7 +331,7 @@ public class SaasMigrationService implements TenantSyncService {
private void finalizeMigration() {
if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
// automaticAnalysisJob.startForTenant(TenantContext.getTenantId());
// automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); // AutomaticAnalysisJob should be re-enabled by re-starting the persistence service pod after a rule change
tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build());
log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
}

13
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -403,7 +403,10 @@ public class FileStatusPersistenceService {
public List<FileEntity> getStatusesForDossierTemplate(String dossierTemplateId) {
return dossierService.findAllDossiersForDossierTemplateId(dossierTemplateId)
.stream().map(dossier -> fileRepository.findByDossierId(dossier.getId())).flatMap(List::stream).toList();
.stream()
.map(dossier -> fileRepository.findByDossierId(dossier.getId()))
.flatMap(List::stream)
.toList();
}
@ -444,6 +447,12 @@ public class FileStatusPersistenceService {
public void hardDelete(String fileId) {
var now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
hardDelete(fileId, now);
}
@Transactional
public void hardDelete(String fileId, OffsetDateTime now) {
fileRepository.hardDeleteFiles(List.of(fileId), ProcessingStatus.PROCESSED, now);
fileAttributesRepository.deleteByFileId(fileId);
@ -453,7 +462,7 @@ public class FileStatusPersistenceService {
@Transactional
public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
}