RED-7834: fix migration issues

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasAnnotationIdMigrationService.java

@@ -5,49 +5,34 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
 import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
-import jakarta.transaction.Transactional;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
 
-import java.util.Map;
+import jakarta.transaction.Transactional;
-import java.util.concurrent.atomic.AtomicInteger;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Service;
 
 @Slf4j
 @Service
+@Transactional
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SaasAnnotationIdMigrationService {
 
-    private final ManualRedactionRepository manualRedactionRepository;
+    ManualRedactionRepository manualRedactionRepository;
-    private final RemoveRedactionRepository removeRedactionRepository;
+    RemoveRedactionRepository removeRedactionRepository;
-    private final ForceRedactionRepository forceRedactionRepository;
+    ForceRedactionRepository forceRedactionRepository;
-    private final ResizeRedactionRepository resizeRedactionRepository;
+    ResizeRedactionRepository resizeRedactionRepository;
-    private final RecategorizationRepository recategorizationRepository;
+    RecategorizationRepository recategorizationRepository;
-    private final LegalBasisChangeRepository legalBasisChangeRepository;
+    LegalBasisChangeRepository legalBasisChangeRepository;
-    private final CommentRepository commentRepository;
+    CommentRepository commentRepository;
-    private final FileRepository fileRepository;
+    FileRepository fileRepository;
 
 
-    @Transactional
+    public int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
-    public void updateAnnotationIds(String fileId, Map<String, String> oldToNewMapping) {
 
-        AtomicInteger numUpdates = new AtomicInteger(0);
-        AtomicInteger numCommentUpdates = new AtomicInteger(0);
-        oldToNewMapping.forEach((key, value) -> {
-            AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key);
-            AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value);
-            numUpdates.getAndAdd(updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numUpdates.getAndAdd(updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
-            numCommentUpdates.getAndAdd(commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value));
-        });
-        log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId);
-    }
-
-    private int updateManualAddRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
         var oldEntry = manualRedactionRepository.findById(oldAnnotationEntityId);
         if (oldEntry.isPresent()) {
 
@@ -64,7 +49,8 @@ public class SaasAnnotationIdMigrationService {
     }
 
 
-    private int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+    public int updateRemoveRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+
         var oldEntry = removeRedactionRepository.findById(oldAnnotationEntityId);
         if (oldEntry.isPresent()) {
 
@@ -80,7 +66,8 @@ public class SaasAnnotationIdMigrationService {
     }
 
 
-    private int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+    public int updateForceRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+
         var oldEntry = forceRedactionRepository.findById(oldAnnotationEntityId);
         if (oldEntry.isPresent()) {
 
@@ -96,7 +83,8 @@ public class SaasAnnotationIdMigrationService {
     }
 
 
-    private int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+    public int updateResizeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+
         var oldEntry = resizeRedactionRepository.findById(oldAnnotationEntityId);
         if (oldEntry.isPresent()) {
 
@@ -113,7 +101,8 @@ public class SaasAnnotationIdMigrationService {
     }
 
 
-    private int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+    public int updateRecategorizationRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+
         var oldEntry = recategorizationRepository.findById(oldAnnotationEntityId);
         if (oldEntry.isPresent()) {
 
@@ -129,7 +118,8 @@ public class SaasAnnotationIdMigrationService {
     }
 
 
-    private int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+    public int updateLegalBasisChangeRedaction(AnnotationEntityId oldAnnotationEntityId, AnnotationEntityId newAnnotationEntityId) {
+
         var oldEntry = legalBasisChangeRepository.findById(oldAnnotationEntityId);
         if (oldEntry.isPresent()) {
 
@@ -145,8 +135,9 @@ public class SaasAnnotationIdMigrationService {
     }
 
 
-    private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
+    public int updateCommentIds(String fileId, String key, String value) {
-        return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build();
+
+        return commentRepository.saasMigrationUpdateAnnotationIds(fileId, key, value);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java

@@ -1,12 +1,14 @@
 package com.iqser.red.service.persistence.management.v1.processor.migration;
 
+import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
 import com.iqser.red.service.persistence.management.v1.processor.service.job.AutomaticAnalysisJob;
 import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
-import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
@@ -31,6 +33,8 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE;
 import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
@@ -79,15 +83,32 @@ public class SaasMigrationService implements TenantSyncService {
         var dossiers = dossierService.getAllDossiers().stream().filter(dossier -> dossier.getHardDeletedTime() == null).toList();
         for (var dossier : dossiers) {
             var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()).stream().filter(file -> file.getHardDeletedTime() == null).toList();
+            var migrationStati = saasMigrationStatusPersistenceService.findAll()
+                    .stream()
+                    .collect(Collectors.toMap(SaasMigrationStatusEntity::getFileId, SaasMigrationStatusEntity::getStatus));
             for (var file : files) {
-                saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossier.getId(), file.getId());
+                if (notExistsOrError(file, migrationStati)) {
-                var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getId(), file.getId(), false);
+                    saasMigrationStatusPersistenceService.createMigrationRequiredStatus(dossier.getId(), file.getId());
-                rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
+                    var layoutParsingRequest = layoutParsingRequestFactory.build(dossier.getId(), file.getId(), false);
-                numberOfFiles++;
+                    rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
+                    numberOfFiles++;
+                } else {
+                    log.info("Skipping file with id {} and dossier {}, since it is already migrated or is currently being migrated!", file.getId(), file.getDossierId());
+                }
             }
         }
 
         log.info("Added {} documents for tenant {} to Layout-Parsing queue for saas migration", numberOfFiles, TenantContext.getTenantId());
+
+        if (numberOfFiles == 0) {
+            finalizeMigration();
+        }
+    }
+
+
+    private boolean notExistsOrError(FileEntity file, Map<String, SaasMigrationStatus> migrationStati) {
+
+        return migrationStati.getOrDefault(file.getId(), SaasMigrationStatus.ERROR).equals(SaasMigrationStatus.ERROR);
     }
 
 
@@ -130,7 +151,6 @@ public class SaasMigrationService implements TenantSyncService {
         } catch (Exception e) {
             log.error("Queuing of entityLog migration failed with {}", e.getMessage());
             saasMigrationStatusPersistenceService.updateErrorStatus(fileId, String.format("Queuing of entityLog migration failed with %s", e.getMessage()));
-
         }
 
     }
@@ -187,15 +207,26 @@ public class SaasMigrationService implements TenantSyncService {
         MigratedIds migratedIds = getMigratedIds(dossierId, fileId);
         Map<String, String> oldToNewMapping = migratedIds.buildOldToNewMapping();
         try {
-            saasAnnotationIdMigrationService.updateAnnotationIds(fileId, oldToNewMapping);
+            updateAnnotationIds(fileId, oldToNewMapping);
         } catch (Exception e) {
-            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, e.getMessage());
+            String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s",
-            log.error("Error during annotation id migration for tenant {} dossier {} and file {}, cause {}", TenantContext.getTenantId(), dossierId, fileId, e.getMessage());
+                    TenantContext.getTenantId(),
+                    dossierId,
+                    fileId,
+                    e.getMessage());
+            saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message);
+            log.error(message);
             throw e;
         }
         saasMigrationStatusPersistenceService.updateStatus(fileId, SaasMigrationStatus.FINISHED);
 
         log.info("AnnotationIds migration finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
+        finalizeMigration();
+    }
+
+
+    private void finalizeMigration() {
+
         if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
             automaticAnalysisJob.startForTenant(TenantContext.getTenantId());
             log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
@@ -203,6 +234,31 @@ public class SaasMigrationService implements TenantSyncService {
     }
 
 
+    public void updateAnnotationIds(String fileId, Map<String, String> oldToNewMapping) {
+
+        AtomicInteger numUpdates = new AtomicInteger(0);
+        AtomicInteger numCommentUpdates = new AtomicInteger(0);
+        oldToNewMapping.forEach((key, value) -> {
+            AnnotationEntityId oldAnnotationEntityId = buildAnnotationId(fileId, key);
+            AnnotationEntityId newAnnotationEntityId = buildAnnotationId(fileId, value);
+            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateManualAddRedaction(oldAnnotationEntityId, newAnnotationEntityId));
+            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRemoveRedaction(oldAnnotationEntityId, newAnnotationEntityId));
+            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateForceRedaction(oldAnnotationEntityId, newAnnotationEntityId));
+            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateResizeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
+            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateRecategorizationRedaction(oldAnnotationEntityId, newAnnotationEntityId));
+            numUpdates.getAndAdd(saasAnnotationIdMigrationService.updateLegalBasisChangeRedaction(oldAnnotationEntityId, newAnnotationEntityId));
+            numCommentUpdates.getAndAdd(saasAnnotationIdMigrationService.updateCommentIds(fileId, key, value));
+        });
+        log.info("Migrated {} annotationIds and {} comments for file {}", numUpdates.get(), numCommentUpdates, fileId);
+    }
+
+
+    private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
+
+        return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build();
+    }
+
+
     private MigratedIds getMigratedIds(String dossierId, String fileId) {
 
         try {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/SaasMigrationStatusPersistenceService.java

@@ -4,23 +4,27 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.migratio
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SaasMigrationStatusRepository;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus;
+
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
+
 import org.springframework.stereotype.Service;
 
 import java.util.List;
 
-
 @Service
 @RequiredArgsConstructor
 public class SaasMigrationStatusPersistenceService {
 
     private final SaasMigrationStatusRepository saasMigrationStatusRepository;
 
+
     public List<SaasMigrationStatusEntity> findAllByStatus(SaasMigrationStatus status) {
+
         return saasMigrationStatusRepository.findAllByStatus(status);
     }
 
+
     public SaasMigrationStatusEntity findById(String fileId) {
 
         var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId);
@@ -30,40 +34,57 @@ public class SaasMigrationStatusPersistenceService {
         throw new NotFoundException("No migration entry found for fileId" + fileId);
     }
 
+
     public boolean isMigrating(String fileId) {
+
         var migrationStatusOptional = saasMigrationStatusRepository.findById(fileId);
         return migrationStatusOptional.isPresent() && migrationStatusOptional.get().getStatus() != SaasMigrationStatus.FINISHED;
     }
 
+
     @Transactional
     public void createMigrationRequiredStatus(String dossierId, String fileId) {
-        saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder()
+
-                .fileId(fileId)
+        saasMigrationStatusRepository.save(SaasMigrationStatusEntity.builder().fileId(fileId).dossierId(dossierId).status(SaasMigrationStatus.MIGRATION_REQUIRED).build());
-                .dossierId(dossierId)
-                .status(SaasMigrationStatus.MIGRATION_REQUIRED)
-                .build());
     }
 
+
     @Transactional
     public void updateStatus(String fileId, SaasMigrationStatus status) {
+
         saasMigrationStatusRepository.updateStatus(fileId, status);
     }
 
+
     @Transactional
     public void updateErrorStatus(String fileId, String errorCause) {
+
         saasMigrationStatusRepository.updateErrorStatus(fileId, SaasMigrationStatus.ERROR, errorCause);
     }
 
+
     @Transactional
     public void updateErrorCounter(String fileId, Integer processingErrorCounter, String errorCause) {
+
         saasMigrationStatusRepository.updateErrorCounter(fileId, processingErrorCounter, errorCause);
     }
 
+
     public int countByStatus(SaasMigrationStatus status) {
+
         return saasMigrationStatusRepository.countByStatus(status);
     }
 
+
     public int countAll() {
+
         return saasMigrationStatusRepository.countAll();
     }
+
+
+    public List<SaasMigrationStatusEntity> findAll() {
+
+        return saasMigrationStatusRepository.findAll();
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java

@@ -64,7 +64,9 @@ public class LayoutParsingFinishedMessageReceiver {
         LayoutParsingRequest analyzeRequest = objectMapper.readValue(failedMessage.getBody(), LayoutParsingRequest.class);
         log.info("Failed to process analyze request: {}", analyzeRequest);
         String errorCause = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER);
-
+        if (errorCause == null) {
+            errorCause = "Error occured during layout parsing!";
+        }
         if (saasMigrationStatusPersistenceService.isMigrating(layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()))) {
             saasMigrationService.handleError(layoutParsingRequestIdentifierService.parseDossierId(analyzeRequest.identifier()),
                     layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()),

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionServiceSaasMigrationMessageReceiver.java

@@ -42,6 +42,9 @@ public class RedactionServiceSaasMigrationMessageReceiver {
 
         var migrationRequest = objectMapper.readValue(failedMessage.getBody(), MigrationRequest.class);
         String errorCause = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
+        if (errorCause == null) {
+            errorCause = "Error occured during entityLog migration!";
+        }
         saasMigrationService.handleError(migrationRequest.getDossierId(), migrationRequest.getFileId(), errorCause, MIGRATION_QUEUE);
 
     }

persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml

@@ -9,7 +9,8 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/
 
 logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
 
-logging.type : ${LOGGING_TYPE:CONSOLE}
+
+logging.type: ${LOGGING_TYPE:CONSOLE}
 kubernetes.namespace: ${NAMESPACE:default}
 project.version: 1.0-SNAPSHOT