From b2cf55b5cc23660c58d41d69a64d9b083524d9fc Mon Sep 17 00:00:00 2001 From: Corina Olariu Date: Thu, 14 Mar 2024 12:20:17 +0200 Subject: [PATCH] RED-8727 - Add rank de-duplication to migration - add RankDeDuplicationService and RankDeduplicationMigration16 to update the found duplicate ranks - junit tests added Signed-off-by: Corina Olariu --- .../migration/RankDeDuplicationService.java | 93 ++++++++++++ .../migration/SaasMigrationService.java | 27 ++-- .../RankDeDuplicationMigration16.java | 41 ++++++ ...eCloneAndExportWithDuplicateRanksTest.java | 139 ++++++++++++++++++ 4 files changed, 288 insertions(+), 12 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/RankDeDuplicationService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RankDeDuplicationMigration16.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/RankDeDuplicationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/RankDeDuplicationService.java new file mode 100644 index 000000000..97cdb2b71 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/RankDeDuplicationService.java @@ -0,0 +1,93 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class RankDeDuplicationService { + + private final DossierTemplateManagementService dossierTemplateManagementService; + + private final DictionaryPersistenceService dictionaryPersistenceService; + + + @SneakyThrows + public void deduplicate() { + + dossierTemplateManagementService.getAllDossierTemplates() + .stream() + .map(DossierTemplate::getId) + .forEach(this::deduplicate); + } + + + private void deduplicate(String dossierTemplateId) { + + List allDossierTemplateTypes = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false); + TreeMap> typesSortedByRank = new TreeMap<>(allDossierTemplateTypes.stream() + .collect(Collectors.groupingBy(TypeEntity::getRank, Collectors.toList()))); + + var lastRankOk = -1; + // we create a map with every rank that needs to be updated and the value is the starting point for that rank + Map rankToNewRankUpdateMap = new TreeMap<>(Collections.reverseOrder()); + for (var typesByRankEntry : typesSortedByRank.entrySet()) { + log.debug(" {} - {}", typesByRankEntry.getKey(), typesByRankEntry.getValue().size()); + typesByRankEntry.getValue() + .forEach(t -> log.debug("type: {} - dtId: {} - dId: {}", t.getType(), t.getDossierTemplateId(), t.getDossierId())); + + if (typesByRankEntry.getValue().size() > 1) { + if (typesByRankEntry.getKey() > lastRankOk) { + rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), typesByRankEntry.getKey()); + lastRankOk = typesByRankEntry.getKey() + typesByRankEntry.getValue().size() - 1; + } else { + rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), lastRankOk + 1); + lastRankOk = lastRankOk + typesByRankEntry.getValue().size(); + } + } else { + if (typesByRankEntry.getKey() > lastRankOk) { + lastRankOk = typesByRankEntry.getKey(); + } else { + lastRankOk = lastRankOk + 1; + rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), lastRankOk); + } + } + } + // need to update in reverse order so we don't get exception for duplicate ranks + for (var rankToUpdateEntry : rankToNewRankUpdateMap.entrySet()) { + var newRank = rankToUpdateEntry.getValue(); + for (TypeEntity t : typesSortedByRank.get(rankToUpdateEntry.getKey())) { + + if (newRank != t.getRank()) { + log.info("Type {} with rank: {} will be updated to rank: {}", t.getType(), t.getRank(), newRank); + t.setRank(newRank); + dictionaryPersistenceService.updateType(t.getId(), t); + var dossierTypes = dictionaryPersistenceService.getAllDossierTypesForDossierTemplateAndType(dossierTemplateId, t.getType(), false); + + for (TypeEntity td : dossierTypes) { + td.setRank(newRank); + dictionaryPersistenceService.updateType(td.getId(), td); + } + + } + newRank = newRank + 1; + } + } + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index fbc218fb1..cf8911208 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -1,5 +1,17 @@ package com.iqser.red.service.persistence.management.v1.processor.migration; +import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE; +import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity; @@ -36,18 +48,6 @@ import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE; -import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; - @Slf4j @Service @RequiredArgsConstructor @@ -68,6 +68,7 @@ public class SaasMigrationService implements TenantSyncService { SaasAnnotationIdMigrationService saasAnnotationIdMigrationService; UncompressedFilesMigrationService uncompressedFilesMigrationService; ManualRedactionService manualRedactionService; + RankDeDuplicationService rankDeDuplicationService; @Override @@ -88,6 +89,8 @@ public class SaasMigrationService implements TenantSyncService { uncompressedFilesMigrationService.migrateUncompressedFiles(tenantId); log.info("Finished uncompressed files migration ..."); + rankDeDuplicationService.deduplicate(); + int numberOfFiles = 0; var dossiers = dossierService.getAllDossiers() .stream() diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RankDeDuplicationMigration16.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RankDeDuplicationMigration16.java new file mode 100644 index 000000000..c85914482 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RankDeDuplicationMigration16.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; +import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Setter +@Service +public class RankDeDuplicationMigration16 extends Migration { + + @Autowired + private RankDeDuplicationService rankDeDuplicationService; + + + public RankDeDuplicationMigration16() { + + super(NAME, VERSION); + } + + + private static final String NAME = "Adding to the migration the rank de-duplication"; + private static final long VERSION = 16; + + + @Override + /* + * In cases were we have duplicate ranks for entities, this needs to be fixed + */ protected void migrate() { + + log.info("Migration: Checking for duplicate ranks"); + rankDeDuplicationService.deduplicate(); + + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java index 7e96b2ace..ec5c1ba81 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java @@ -1,8 +1,13 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; +import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; @@ -10,12 +15,16 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.junit.jupiter.SpringExtension; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService; @@ -84,11 +93,17 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { private DossierTemplatePersistenceService dossierTemplatePersistenceService; private DossierTemplateCloneService dossierTemplateCloneService; private DossierTemplateExportService dossierTemplateExportService; + private RankDeDuplicationService rankDeDuplicationService; private DossierTemplateManagementService dossierTemplateManagementService; private String dossierTemplateId = "dossierTemplateId"; + + private String dossierId = "dossierId"; private String dossierTemplateName = "Clone of " + dossierTemplateId; + @Captor + private ArgumentCaptor captor; + @BeforeEach public void setupData() { @@ -131,6 +146,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { dossierTemplatePersistenceService, dossierTemplateCloneService); + rankDeDuplicationService = new RankDeDuplicationService(dossierTemplateManagementService, dictionaryPersistenceService); + List typesValues = new ArrayList<>(); TypeRankSummary typeRank1 = new TypeRankSummary(dossierTemplateId, null, 50, 2); TypeRankSummary typeRank2 = new TypeRankSummary(dossierTemplateId, null, 100, 1); @@ -165,4 +182,126 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest { } + + @Test + @SneakyThrows + public void testDeduplicateDossierTemplate() { + + List allDossierTemplateTypes = new ArrayList<>(); + allDossierTemplateTypes.add(createTypeEntity("type00", dossierTemplateId, 0)); // no change + allDossierTemplateTypes.add(createTypeEntity("type01", dossierTemplateId, 0)); // new rank will be 1 + allDossierTemplateTypes.add(createTypeEntity("type02", dossierTemplateId, 0)); // new rank will be 2 + allDossierTemplateTypes.add(createTypeEntity("type22", dossierTemplateId, 2)); // new rank will be 3 + allDossierTemplateTypes.add(createTypeEntity("type0", dossierTemplateId, 49)); // no change + allDossierTemplateTypes.add(createTypeEntity("type1", dossierTemplateId, 50)); // no change + allDossierTemplateTypes.add(createTypeEntity("type2", dossierTemplateId, 50)); // new rank will be 51 + allDossierTemplateTypes.add(createTypeEntity("type3", dossierTemplateId, 50)); // new rank will be 52 + allDossierTemplateTypes.add(createTypeEntity("type4", dossierTemplateId, 50)); // new rank will be 53 + allDossierTemplateTypes.add(createTypeEntity("type5", dossierTemplateId, 50)); // new rank will be 54 + allDossierTemplateTypes.add(createTypeEntity("type6", dossierTemplateId, 51)); // new rank will be 55 + allDossierTemplateTypes.add(createTypeEntity("type7", dossierTemplateId, 51)); // new rank will be 55 + allDossierTemplateTypes.add(createTypeEntity("type8", dossierTemplateId, 51)); // new rank will be 57 + allDossierTemplateTypes.add(createTypeEntity("type9", dossierTemplateId, 52)); // new rank will be 58 + allDossierTemplateTypes.add(createTypeEntity("type10", dossierTemplateId, 52)); // new rank will be 59 + allDossierTemplateTypes.add(createTypeEntity("type11", dossierTemplateId, 55)); // new rank will be 60 + allDossierTemplateTypes.add(createTypeEntity("type12", dossierTemplateId, 100)); // no change + + List dossierTemplates = new ArrayList<>(); + DossierTemplateEntity dt = new DossierTemplateEntity(); + dt.setId(dossierTemplateId); + dossierTemplates.add(dt); + when(dossierTemplateRepository.findAllWhereDeletedIsFalse()).thenReturn(dossierTemplates); + when(legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId)).thenReturn(Collections.emptyList()); + when(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false)).thenReturn(allDossierTemplateTypes); + when(dictionaryPersistenceService.getAllDossierTypesForDossierTemplateAndType(dossierTemplateId, "type02", false)).thenReturn(List.of(createTypeEntity("type02", + dossierTemplateId, + dossierId, + 0))); + + rankDeDuplicationService.deduplicate(); + + TypeEntity caughtType; + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type01", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 1); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type02", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 2); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type02", dossierTemplateId, dossierId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 2); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type22", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 3); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type2", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 51); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type3", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 52); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type4", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 53); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type5", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 54); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type6", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 55); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type7", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 56); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type8", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 57); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type9", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 58); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type10", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 59); + + verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type11", dossierTemplateId)), captor.capture()); + caughtType = captor.getValue(); + Assertions.assertEquals(caughtType.getRank(), 60); + + } + + + private TypeEntity createTypeEntity(String type, String dossierTemplateId, int rank) { + + return createTypeEntity(type, dossierTemplateId, null, rank); + } + + + private TypeEntity createTypeEntity(String type, String dossierTemplateId, String dossierId, int rank) { + + TypeEntity typeEntity = new TypeEntity(); + typeEntity.setId(toTypeId(type, dossierTemplateId, dossierId)); + typeEntity.setType(type); + typeEntity.setDossierTemplateId(dossierTemplateId); + typeEntity.setRank(rank); + typeEntity.setDescription("description"); + typeEntity.setLabel(type); + typeEntity.setHexColor("#ddddd"); + typeEntity.setRecommendationHexColor("#cccccc"); + typeEntity.setSkippedHexColor("#cccccc"); + typeEntity.setAddToDictionaryAction(true); + + return typeEntity; + } + } -- 2.47.2