Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-3818: Added more logging for false_positve migration, disabled scheduling during migration

Browse Source
This commit is contained in:
deiflaender 2022-04-08 13:46:57 +02:00
parent cf4da4cd39
commit 79a7bfbdd0
2 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java
View File

@ -2,24 +2,17 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
import com.iqser.red.service.peristence.v1.server.controller.DossierTemplateController;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
@ -64,6 +57,7 @@ public class TypeToEntityMigration5 extends Migration {
var dossierTemplates = dossierTemplateRepository.findAllWhereDeletedIsFalse();
dossierTemplates.forEach(dossierTemplate -> {
log.info("Starting false positive migration of dossierTemplate {}", dossierTemplate.getId());
var typeIdsToDelete = new HashSet<String>();
var typesOfDossierTemplate = dictionaryPersistenceService.getAllTypes();
var falsePositive = typesOfDossierTemplate.stream()
@ -78,11 +72,14 @@ public class TypeToEntityMigration5 extends Migration {
.stream()
.map(f -> f.getValue())
.collect(Collectors.toSet());
log.info("False positive migration of dossierTemplate {} has {} false positive entries", dossierTemplate.getId(), falsePositiveEntries.size());
typesOfDossierTemplate.stream().filter(t -> !t.getType().equals("false_positive")).forEach(typeEntity -> {
log.info("Start processing type {}", typeEntity.getType());
if (typeEntity.getType().startsWith("recommendation_")) {
typeIdsToDelete.add(typeEntity.getId());
log.info("Finished processing type {}", typeEntity.getType());
return;
}
@ -90,6 +87,7 @@ public class TypeToEntityMigration5 extends Migration {
.stream()
.map(f -> f.getValue())
.collect(Collectors.toSet());
log.info("type {} has {} entries", typeEntity.getType(), entries);
var typeFalsePositives = new HashSet<String>();
falsePositiveEntries.forEach(falsePositiveValue -> {
@ -100,14 +98,15 @@ public class TypeToEntityMigration5 extends Migration {
if (!typeFalsePositives.isEmpty()) {
dictionaryController.addEntries(typeEntity.getId(), new ArrayList<>(typeFalsePositives), false, true, DictionaryEntryType.FALSE_POSITIVE);
log.info("Added {} for type {}", typeFalsePositives.size(), typeEntity.getType());
}
log.info("Finished processing type {}", typeEntity.getType());
});
typeIdsToDelete.forEach(typeIdToDelete -> {
dictionaryController.deleteType(typeIdToDelete);
});
log.info("Finished false positive migration of dossierTemplate {}", dossierTemplate.getId());
});
}

7
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.peristence.v1.server.service.scheduler;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.service.ReanalysisRequiredStatusService;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import lombok.RequiredArgsConstructor;
@ -29,6 +30,7 @@ public class AutomaticAnalysisScheduler {
private int pageFactor;
private final FileStatusService fileStatusService;
private final AmqpAdmin amqpAdmin;
private final FileManagementServiceSettings fileManagementServiceSettings;
@PostConstruct
@ -42,6 +44,11 @@ public class AutomaticAnalysisScheduler {
@Scheduled(fixedDelay = 10000, initialDelay = 10000)
public void checkFilesThatRequireReanalysisAndQueueIfPossible() {
if(fileManagementServiceSettings.isMigrateOnly()){
log.info("Skipping scheduling during migration");
return;
}
var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
if (redactionQueueInfo != null) {
log.info("Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", MessagingConfiguration.REDACTION_QUEUE, redactionQueueInfo