From 1ea65a256c582f3f9dd3a3ff0b2e49dd740c8e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Wed, 20 Dec 2023 12:28:46 +0100 Subject: [PATCH] RED-7834: fix migration issues --- .../SaasAnnotationIdMigrationService.java | 71 ++++++++---------- .../migration/SaasMigrationService.java | 74 ++++++++++++++++--- ...SaasMigrationStatusPersistenceService.java | 33 +++++++-- .../LayoutParsingFinishedMessageReceiver.java | 4 +- ...onServiceSaasMigrationMessageReceiver.java | 3 + .../src/main/resources/application.yaml | 3 +- 6 files changed, 131 insertions(+), 57 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java index 67bd1dbdf..f9fa0ec93 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java @@ -5,49 +5,34 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*; import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; -import jakarta.transaction.Transactional; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; +import jakarta.transaction.Transactional; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Service; @Slf4j @Service +@Transactional @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class SaasAnnotationIdMigrationService { - private final ManualRedactionRepository manualRedactionRepository; - private final RemoveRedactionRepository removeRedactionRepository; - private final ForceRedactionRepository forceRedactionRepository; - private final ResizeRedactionRepository resizeRedactionRepository; - private final RecategorizationRepository recategorizationRepository; - private final LegalBasisChangeRepository legalBasisChangeRepository; - private final CommentRepository commentRepository; - private final FileRepository fileRepository; + ManualRedactionRepository manualRedactionRepository; + RemoveRedactionRepository removeRedactionRepository; + ForceRedactionRepository forceRedactionRepository; + ResizeRedactionRepository resizeRedactionRepository; + RecategorizationRepository recategorizationRepository; + LegalBasisChangeRepository legalBasisChangeRepository; + CommentRepository commentRepository; + FileRepository fileRepository; - @Transactional - public void updateAnnotationIds(String fileId, Map oldToNewMapping) { + public int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { - AtomicInteger numUpdates = new AtomicInteger(0); - AtomicInteger numCommentUpdates = new AtomicInteger(0); - oldToNewMapping.forEach((key, value) -> { - AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key); - AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value); - numUpdates.getAndAdd(updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numUpdates.getAndAdd(updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId)); - numCommentUpdates.getAndAdd(commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value)); - }); - log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId); - } - - private int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { var oldEntry = manualRedactionRepository.findById(oldAnnotationEntityId); if (oldEntry.isPresent()) { @@ -64,7 +49,8 @@ public class SaasAnnotationIdMigrationService { } - private int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + public int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + var oldEntry = removeRedactionRepository.findById(oldAnnotationEntityId); if (oldEntry.isPresent()) { @@ -80,7 +66,8 @@ public class SaasAnnotationIdMigrationService { } - private int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + public int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + var oldEntry = forceRedactionRepository.findById(oldAnnotationEntityId); if (oldEntry.isPresent()) { @@ -96,7 +83,8 @@ public class SaasAnnotationIdMigrationService { } - private int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + public int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + var oldEntry = resizeRedactionRepository.findById(oldAnnotationEntityId); if (oldEntry.isPresent()) { @@ -113,7 +101,8 @@ public class SaasAnnotationIdMigrationService { } - private int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + public int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + var oldEntry = recategorizationRepository.findById(oldAnnotationEntityId); if (oldEntry.isPresent()) { @@ -129,7 +118,8 @@ public class SaasAnnotationIdMigrationService { } - private int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + public int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) { + var oldEntry = legalBasisChangeRepository.findById(oldAnnotationEntityId); if (oldEntry.isPresent()) { @@ -145,8 +135,9 @@ public class SaasAnnotationIdMigrationService { } - private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) { - return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build(); + public int updateCommentIds(String fileId, String key, String value) { + + return commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 11327ffc3..5b57a0093 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -1,12 +1,14 @@ package com.iqser.red.service.persistence.management.v1.processor.migration; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; -import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; @@ -31,6 +33,8 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE; import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; @@ -79,15 +83,32 @@ public class SaasMigrationService implements TenantSyncService { var dossiers = dossierService.getAllDossiers().stream().filter(dossier -> dossier.getHardDeletedTime() == null).toList(); for (var dossier : dossiers) { var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()).stream().filter(file -> file.getHardDeletedTime() == null).toList(); + var migrationStati = saasMigrationStatusPersistenceService.findAll() + .stream() + .collect(Collectors.toMap(SaasMigrationStatusEntity::getFileId, SaasMigrationStatusEntity::getStatus)); for (var file : files) { - saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossier.getId(), file.getId()); - var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getId(), file.getId(), false); - rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest); - numberOfFiles++; + if (notExistsOrError(file, migrationStati)) { + saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossier.getId(), file.getId()); + var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getId(), file.getId(), false); + rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest); + numberOfFiles++; + } else { + log.info("Skipping file with id {} and dossier {}, since it is already migrated or is currently being migrated!", file.getId(), file.getDossierId()); + } } } log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId()); + + if (numberOfFiles == 0) { + finalizeMigration(); + } + } + + + private boolean notExistsOrError(FileEntity file, Map migrationStati) { + + return migrationStati.getOrDefault(file.getId(), SaasMigrationStatus.ERROR).equals(SaasMigrationStatus.ERROR); } @@ -130,7 +151,6 @@ public class SaasMigrationService implements TenantSyncService { } catch (Exception e) { log.error("Queuing of entityLog migration failed with {}", e.getMessage()); saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage())); - } } @@ -187,15 +207,26 @@ public class SaasMigrationService implements TenantSyncService { MigratedIds migratedIds = getMigratedIds(dossierId, fileId); Map oldToNewMapping = migratedIds.buildOldToNewMapping(); try { - saasAnnotationIdMigrationService.updateAnnotationIds(fileId, oldToNewMapping); + updateAnnotationIds(fileId, oldToNewMapping); } catch (Exception e) { - saasMigrationStatusPersistenceService.updateErrorStatus(fileId, e.getMessage()); - log.error("Error during annotation id migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, e.getMessage()); + String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s", + TenantContext.getTenantId(), + dossierId, + fileId, + e.getMessage()); + saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message); + log.error(message); throw e; } saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED); log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId); + finalizeMigration(); + } + + + private void finalizeMigration() { + if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) { automaticAnalysisJob.startForTenant(TenantContext.getTenantId()); log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId()); @@ -203,6 +234,31 @@ public class SaasMigrationService implements TenantSyncService { } + public void updateAnnotationIds(String fileId, Map oldToNewMapping) { + + AtomicInteger numUpdates = new AtomicInteger(0); + AtomicInteger numCommentUpdates = new AtomicInteger(0); + oldToNewMapping.forEach((key, value) -> { + AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key); + AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value); + numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId)); + numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId)); + numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId)); + numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId)); + numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId)); + numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId)); + numCommentUpdates.getAndAdd(saasAnnotationIdMigrationService.updateCommentIds(fileId, key, value)); + }); + log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId); + } + + + private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) { + + return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build(); + } + + private MigratedIds getMigratedIds(String dossierId, String fileId) { try { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java index e49895a98..b6bb7ee43 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java @@ -4,23 +4,27 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.migratio import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus; + import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; + import org.springframework.stereotype.Service; import java.util.List; - @Service @RequiredArgsConstructor public class SaasMigrationStatusPersistenceService { private final SaasMigrationStatusRepository saasMigrationStatusRepository; + public List findAllByStatus(SaasMigrationStatus status) { + return saasMigrationStatusRepository.findAllByStatus(status); } + public SaasMigrationStatusEntity findById(String fileId) { var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId); @@ -30,40 +34,57 @@ public class SaasMigrationStatusPersistenceService { throw new NotFoundException("No migration entry found for fileId" + fileId); } + public boolean isMigrating(String fileId) { + var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId); return migrationStatusOptional.isPresent() && migrationStatusOptional.get().getStatus() != SaasMigrationStatus.FINISHED; } + @Transactional public void createMigrationRequiredStatus(String dossierId, String fileId) { - saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder() - .fileId(fileId) - .dossierId(dossierId) - .status(SaasMigrationStatus.MIGRATION_REQUIRED) - .build()); + + saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder().fileId(fileId).dossierId(dossierId).status(SaasMigrationStatus.MIGRATION_REQUIRED).build()); } + @Transactional public void updateStatus(String fileId, SaasMigrationStatus status) { + saasMigrationStatusRepository.updateStatus(fileId, status); } + @Transactional public void updateErrorStatus(String fileId, String errorCause) { + saasMigrationStatusRepository.updateErrorStatus(fileId, SaasMigrationStatus.ERROR, errorCause); } + @Transactional public void updateErrorCounter(String fileId, Integer processingErrorCounter, String errorCause) { + saasMigrationStatusRepository.updateErrorCounter(fileId, processingErrorCounter, errorCause); } + public int countByStatus(SaasMigrationStatus status) { + return saasMigrationStatusRepository.countByStatus(status); } + public int countAll() { + return saasMigrationStatusRepository.countAll(); } + + + public List findAll() { + + return saasMigrationStatusRepository.findAll(); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index c9b85a19b..6d9b1b532 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -64,7 +64,9 @@ public class LayoutParsingFinishedMessageReceiver { LayoutParsingRequest analyzeRequest = objectMapper.readValue(failedMessage.getBody(), LayoutParsingRequest.class); log.info("Failed to process analyze request: {}", analyzeRequest); String errorCause = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER); - + if (errorCause == null) { + errorCause = "Error occured during layout parsing!"; + } if (saasMigrationStatusPersistenceService.isMigrating(layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()))) { saasMigrationService.handleError(layoutParsingRequestIdentifierService.parseDossierId(analyzeRequest.identifier()), layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()), diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java index be3d50c6d..0e278b050 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java @@ -42,6 +42,9 @@ public class RedactionServiceSaasMigrationMessageReceiver { var migrationRequest = objectMapper.readValue(failedMessage.getBody(), MigrationRequest.class); String errorCause = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER); + if (errorCause == null) { + errorCause = "Error occured during entityLog migration!"; + } saasMigrationService.handleError(migrationRequest.getDossierId(), migrationRequest.getFileId(), errorCause, MIGRATION_QUEUE); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml index 6b57d5755..0af757fe1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml @@ -9,7 +9,8 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/ logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]" -logging.type : ${LOGGING_TYPE:CONSOLE} + +logging.type: ${LOGGING_TYPE:CONSOLE} kubernetes.namespace: ${NAMESPACE:default} project.version: 1.0-SNAPSHOT