From 79a7bfbdd09fef5a670abb46fefe00c6f8a9f607 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 8 Apr 2022 13:46:57 +0200 Subject: [PATCH] RED-3818: Added more logging for false_positve migration, disabled scheduling during migration --- .../migrations/TypeToEntityMigration5.java | 17 ++++++++--------- .../scheduler/AutomaticAnalysisScheduler.java | 7 +++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java index 21e1a4197..490c7e2fb 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java @@ -2,24 +2,17 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; import java.util.ArrayList; import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import com.iqser.red.service.peristence.v1.server.controller.DictionaryController; -import com.iqser.red.service.peristence.v1.server.controller.DossierTemplateController; import com.iqser.red.service.peristence.v1.server.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; @@ -64,6 +57,7 @@ public class TypeToEntityMigration5 extends Migration { var dossierTemplates = dossierTemplateRepository.findAllWhereDeletedIsFalse(); dossierTemplates.forEach(dossierTemplate -> { + log.info("Starting false positive migration of dossierTemplate {}", dossierTemplate.getId()); var typeIdsToDelete = new HashSet(); var typesOfDossierTemplate = dictionaryPersistenceService.getAllTypes(); var falsePositive = typesOfDossierTemplate.stream() @@ -78,11 +72,14 @@ public class TypeToEntityMigration5 extends Migration { .stream() .map(f -> f.getValue()) .collect(Collectors.toSet()); + log.info("False positive migration of dossierTemplate {} has {} false positive entries", dossierTemplate.getId(), falsePositiveEntries.size()); typesOfDossierTemplate.stream().filter(t -> !t.getType().equals("false_positive")).forEach(typeEntity -> { + log.info("Start processing type {}", typeEntity.getType()); if (typeEntity.getType().startsWith("recommendation_")) { typeIdsToDelete.add(typeEntity.getId()); + log.info("Finished processing type {}", typeEntity.getType()); return; } @@ -90,6 +87,7 @@ public class TypeToEntityMigration5 extends Migration { .stream() .map(f -> f.getValue()) .collect(Collectors.toSet()); + log.info("type {} has {} entries", typeEntity.getType(), entries); var typeFalsePositives = new HashSet(); falsePositiveEntries.forEach(falsePositiveValue -> { @@ -100,14 +98,15 @@ public class TypeToEntityMigration5 extends Migration { if (!typeFalsePositives.isEmpty()) { dictionaryController.addEntries(typeEntity.getId(), new ArrayList<>(typeFalsePositives), false, true, DictionaryEntryType.FALSE_POSITIVE); + log.info("Added {} for type {}", typeFalsePositives.size(), typeEntity.getType()); } - + log.info("Finished processing type {}", typeEntity.getType()); }); typeIdsToDelete.forEach(typeIdToDelete -> { dictionaryController.deleteType(typeIdToDelete); }); - + log.info("Finished false positive migration of dossierTemplate {}", dossierTemplate.getId()); }); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java index f699ba0b2..4f770f6ec 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java @@ -3,6 +3,7 @@ package com.iqser.red.service.peristence.v1.server.service.scheduler; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.service.FileStatusService; import com.iqser.red.service.peristence.v1.server.service.ReanalysisRequiredStatusService; +import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import lombok.RequiredArgsConstructor; @@ -29,6 +30,7 @@ public class AutomaticAnalysisScheduler { private int pageFactor; private final FileStatusService fileStatusService; private final AmqpAdmin amqpAdmin; + private final FileManagementServiceSettings fileManagementServiceSettings; @PostConstruct @@ -42,6 +44,11 @@ public class AutomaticAnalysisScheduler { @Scheduled(fixedDelay = 10000, initialDelay = 10000) public void checkFilesThatRequireReanalysisAndQueueIfPossible() { + if(fileManagementServiceSettings.isMigrateOnly()){ + log.info("Skipping scheduling during migration"); + return; + } + var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); if (redactionQueueInfo != null) { log.info("Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", MessagingConfiguration.REDACTION_QUEUE, redactionQueueInfo