Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-8727 - Add rank de-duplication to migration #398

Merged
corina.olariu.ext1 merged 1 commits from RED-8727 into master 2024-03-14 16:13:28 +01:00
Conversation 4 Commits 1 Files Changed 4 +288 -12
4 changed files with 288 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/RankDeDuplicationService.java Normal file
View File

@ -0,0 +1,93 @@
package com.iqser.red.service.persistence.management.v1.processor.migration;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class RankDeDuplicationService {
private final DossierTemplateManagementService dossierTemplateManagementService;
private final DictionaryPersistenceService dictionaryPersistenceService;
@SneakyThrows
public void deduplicate() {
dossierTemplateManagementService.getAllDossierTemplates()
.stream()
.map(DossierTemplate::getId)
.forEach(this::deduplicate);
}
private void deduplicate(String dossierTemplateId) {
List<TypeEntity> allDossierTemplateTypes = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false);
TreeMap<Integer, List<TypeEntity>> typesSortedByRank = new TreeMap<>(allDossierTemplateTypes.stream()
.collect(Collectors.groupingBy(TypeEntity::getRank, Collectors.toList())));
var lastRankOk = -1;
// we create a map with every rank that needs to be updated and the value is the starting point for that rank
Map<Integer, Integer> rankToNewRankUpdateMap = new TreeMap<>(Collections.reverseOrder());
for (var typesByRankEntry : typesSortedByRank.entrySet()) {
log.debug(" {} - {}", typesByRankEntry.getKey(), typesByRankEntry.getValue().size());
typesByRankEntry.getValue()
.forEach(t -> log.debug("type: {} - dtId: {} - dId: {}", t.getType(), t.getDossierTemplateId(), t.getDossierId()));
if (typesByRankEntry.getValue().size() > 1) {
if (typesByRankEntry.getKey() > lastRankOk) {
rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), typesByRankEntry.getKey());
lastRankOk = typesByRankEntry.getKey() + typesByRankEntry.getValue().size() - 1;
} else {
rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), lastRankOk + 1);
lastRankOk = lastRankOk + typesByRankEntry.getValue().size();
}
} else {
if (typesByRankEntry.getKey() > lastRankOk) {
lastRankOk = typesByRankEntry.getKey();
} else {
lastRankOk = lastRankOk + 1;
rankToNewRankUpdateMap.put(typesByRankEntry.getKey(), lastRankOk);
}
}
}
// need to update in reverse order so we don't get exception for duplicate ranks
for (var rankToUpdateEntry : rankToNewRankUpdateMap.entrySet()) {
var newRank = rankToUpdateEntry.getValue();
for (TypeEntity t : typesSortedByRank.get(rankToUpdateEntry.getKey())) {
if (newRank != t.getRank()) {
log.info("Type {} with rank: {} will be updated to rank: {}", t.getType(), t.getRank(), newRank);
t.setRank(newRank);
dictionaryPersistenceService.updateType(t.getId(), t);
var dossierTypes = dictionaryPersistenceService.getAllDossierTypesForDossierTemplateAndType(dossierTemplateId, t.getType(), false);
for (TypeEntity td : dossierTypes) {
td.setRank(newRank);
dictionaryPersistenceService.updateType(td.getId(), td);
}
}
newRank = newRank + 1;
}
}
}
}

27
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -1,5 +1,17 @@
package com.iqser.red.service.persistence.management.v1.processor.migration;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
@ -36,18 +48,6 @@ import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
@Slf4j
@Service
@RequiredArgsConstructor
@ -68,6 +68,7 @@ public class SaasMigrationService implements TenantSyncService {
SaasAnnotationIdMigrationService saasAnnotationIdMigrationService;
UncompressedFilesMigrationService uncompressedFilesMigrationService;
ManualRedactionService manualRedactionService;
RankDeDuplicationService rankDeDuplicationService;
@Override
@ -88,6 +89,8 @@ public class SaasMigrationService implements TenantSyncService {
uncompressedFilesMigrationService.migrateUncompressedFiles(tenantId);
log.info("Finished uncompressed files migration ...");
rankDeDuplicationService.deduplicate();
int numberOfFiles = 0;
var dossiers = dossierService.getAllDossiers()
.stream()

41
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/RankDeDuplicationMigration16.java Normal file
View File

@ -0,0 +1,41 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
public class RankDeDuplicationMigration16 extends Migration {
@Autowired
private RankDeDuplicationService rankDeDuplicationService;
public RankDeDuplicationMigration16() {
super(NAME, VERSION);
}
private static final String NAME = "Adding to the migration the rank de-duplication";
private static final long VERSION = 16;
@Override
/*
* In cases were we have duplicate ranks for entities, this needs to be fixed
*/ protected void migrate() {
log.info("Migration: Checking for duplicate ranks");
rankDeDuplicationService.deduplicate();
}
}

139
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateCloneAndExportWithDuplicateRanksTest.java
View File

@ -1,8 +1,13 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@ -10,12 +15,16 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateImportService;
@ -84,11 +93,17 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
private DossierTemplatePersistenceService dossierTemplatePersistenceService;
private DossierTemplateCloneService dossierTemplateCloneService;
private DossierTemplateExportService dossierTemplateExportService;
private RankDeDuplicationService rankDeDuplicationService;
private DossierTemplateManagementService dossierTemplateManagementService;
private String dossierTemplateId = "dossierTemplateId";
private String dossierId = "dossierId";
private String dossierTemplateName = "Clone of " + dossierTemplateId;
@Captor
private ArgumentCaptor<TypeEntity> captor;
@BeforeEach
public void setupData() {
@ -131,6 +146,8 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
dossierTemplatePersistenceService,
dossierTemplateCloneService);
rankDeDuplicationService = new RankDeDuplicationService(dossierTemplateManagementService, dictionaryPersistenceService);
List<TypeRankSummary> typesValues = new ArrayList<>();
TypeRankSummary typeRank1 = new TypeRankSummary(dossierTemplateId, null, 50, 2);
TypeRankSummary typeRank2 = new TypeRankSummary(dossierTemplateId, null, 100, 1);
@ -165,4 +182,126 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
}
@Test
@SneakyThrows
public void testDeduplicateDossierTemplate() {
List<TypeEntity> allDossierTemplateTypes = new ArrayList<>();
allDossierTemplateTypes.add(createTypeEntity("type00", dossierTemplateId, 0)); // no change
allDossierTemplateTypes.add(createTypeEntity("type01", dossierTemplateId, 0)); // new rank will be 1
allDossierTemplateTypes.add(createTypeEntity("type02", dossierTemplateId, 0)); // new rank will be 2
allDossierTemplateTypes.add(createTypeEntity("type22", dossierTemplateId, 2)); // new rank will be 3
allDossierTemplateTypes.add(createTypeEntity("type0", dossierTemplateId, 49)); // no change
allDossierTemplateTypes.add(createTypeEntity("type1", dossierTemplateId, 50)); // no change
allDossierTemplateTypes.add(createTypeEntity("type2", dossierTemplateId, 50)); // new rank will be 51
allDossierTemplateTypes.add(createTypeEntity("type3", dossierTemplateId, 50)); // new rank will be 52
allDossierTemplateTypes.add(createTypeEntity("type4", dossierTemplateId, 50)); // new rank will be 53
allDossierTemplateTypes.add(createTypeEntity("type5", dossierTemplateId, 50)); // new rank will be 54
allDossierTemplateTypes.add(createTypeEntity("type6", dossierTemplateId, 51)); // new rank will be 55
allDossierTemplateTypes.add(createTypeEntity("type7", dossierTemplateId, 51)); // new rank will be 55
allDossierTemplateTypes.add(createTypeEntity("type8", dossierTemplateId, 51)); // new rank will be 57
allDossierTemplateTypes.add(createTypeEntity("type9", dossierTemplateId, 52)); // new rank will be 58
allDossierTemplateTypes.add(createTypeEntity("type10", dossierTemplateId, 52)); // new rank will be 59
allDossierTemplateTypes.add(createTypeEntity("type11", dossierTemplateId, 55)); // new rank will be 60
allDossierTemplateTypes.add(createTypeEntity("type12", dossierTemplateId, 100)); // no change
List<DossierTemplateEntity> dossierTemplates = new ArrayList<>();
DossierTemplateEntity dt = new DossierTemplateEntity();
dt.setId(dossierTemplateId);
dossierTemplates.add(dt);
when(dossierTemplateRepository.findAllWhereDeletedIsFalse()).thenReturn(dossierTemplates);
when(legalBasisMappingPersistenceService.getLegalBasisMapping(dossierTemplateId)).thenReturn(Collections.emptyList());
when(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false)).thenReturn(allDossierTemplateTypes);
when(dictionaryPersistenceService.getAllDossierTypesForDossierTemplateAndType(dossierTemplateId, "type02", false)).thenReturn(List.of(createTypeEntity("type02",
dossierTemplateId,
dossierId,
0)));
rankDeDuplicationService.deduplicate();
TypeEntity caughtType;
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type01", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 1);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type02", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 2);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type02", dossierTemplateId, dossierId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 2);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type22", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 3);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type2", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 51);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type3", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 52);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type4", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 53);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type5", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 54);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type6", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 55);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type7", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 56);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type8", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 57);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type9", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 58);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type10", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 59);
verify(dictionaryPersistenceService, times(1)).updateType(eq(toTypeId("type11", dossierTemplateId)), captor.capture());
caughtType = captor.getValue();
Assertions.assertEquals(caughtType.getRank(), 60);
}
private TypeEntity createTypeEntity(String type, String dossierTemplateId, int rank) {
return createTypeEntity(type, dossierTemplateId, null, rank);
}
private TypeEntity createTypeEntity(String type, String dossierTemplateId, String dossierId, int rank) {
TypeEntity typeEntity = new TypeEntity();
typeEntity.setId(toTypeId(type, dossierTemplateId, dossierId));
typeEntity.setType(type);
typeEntity.setDossierTemplateId(dossierTemplateId);
typeEntity.setRank(rank);
typeEntity.setDescription("description");
typeEntity.setLabel(type);
typeEntity.setHexColor("#ddddd");
typeEntity.setRecommendationHexColor("#cccccc");
typeEntity.setSkippedHexColor("#cccccc");
typeEntity.setAddToDictionaryAction(true);
return typeEntity;
}
}