From 26411e1c1c867a35452a0acb878c8b33b37befa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Fri, 16 Aug 2024 14:58:38 +0200 Subject: [PATCH] RED-9902: copy overrides to mongo --- .../migration/StorageToMongoCopyService.java | 112 +++++++++++++++--- .../ComponentOverridesMigration21.java | 34 ++++++ .../DocumineLayoutRewriteMigration20.java | 2 +- .../migrations/StorageToMongoMigration18.java | 2 +- .../service/ComponentLogService.java | 37 ++++-- .../StorageToMongoDBPerformanceTest.java | 3 +- .../service/ComponentLogMongoService.java | 8 +- 7 files changed, 161 insertions(+), 37 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ComponentOverridesMigration21.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java index ac1629bcb..dae219827 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java @@ -1,23 +1,29 @@ package com.iqser.red.service.persistence.management.v1.processor.migration; +import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; +import java.util.function.Consumer; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentsOverrides; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ComponentDocument; import com.iqser.red.storage.commons.exception.StorageException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; @@ -37,30 +43,49 @@ public class StorageToMongoCopyService { private final DossierRepository dossierRepository; private final FileRepository fileRepository; + private final ComponentLogService componentLogService; @SneakyThrows - public void copy() { + public void copyEntityLogs() { + + forEachFile(this::copyEntityLogs); + } + + + public void copyComponentOverrides() { + + forEachFile(this::copyComponentOverrides); + } + + + private void forEachFile(Consumer consumer) { List dossierEntities = dossierRepository.findAll(); List files = new ArrayList<>(); dossierEntities.forEach(dossierEntity -> { List byDossierId = fileRepository.findByDossierId(dossierEntity.getId()); - byDossierId.forEach(file -> files.add(new DossierFile(dossierEntity.getId(), file.getId()))); + byDossierId.forEach(file -> files.add(new DossierFile(dossierEntity.getId(), file.getId(), file.getDeleted(), file.getHardDeletedTime()))); }); for (DossierFile dossierFile : files) { - log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId); - Optional entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId); - if (entityLogFromStorage.isPresent()) { - EntityLog entityLog = removeDuplicates(entityLogFromStorage.get()); - log.info("File found, now saving in mongodb"); - fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId); - fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLog); - log.info("Deleting old file from storage"); - fileManagementStorageService.storeJSONObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG_BAK, entityLogFromStorage.get()); - fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG); - } + consumer.accept(dossierFile); + } + } + + + private void copyEntityLogs(DossierFile dossierFile) { + + log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId); + Optional entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId); + if (entityLogFromStorage.isPresent()) { + EntityLog entityLog = removeDuplicates(entityLogFromStorage.get()); + log.info("File found, now saving in mongodb"); + fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId); + fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLog); + log.info("Deleting old file from storage"); + fileManagementStorageService.storeJSONObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG_BAK, entityLogFromStorage.get()); + fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG); } } @@ -102,7 +127,62 @@ public class StorageToMongoCopyService { } - public record DossierFile(String dossierId, String fileId) { + private void copyComponentOverrides(DossierFile file) { + + if (file.hardDeleted != null) { + return; + } + + Optional optionalComponentsOverrides = getComponentOverridesFromStorageForMigration(file.dossierId(), file.fileId()); + if (optionalComponentsOverrides.isEmpty()) { + return; + } + ComponentsOverrides oldComponentsOverrides = optionalComponentsOverrides.get(); + oldComponentsOverrides.getComponentOverrides() + .forEach((componentName, componentValue) -> { + ComponentLogEntry overrideEntry = buildOverride(componentName, componentValue); + ComponentLogEntry result = componentLogService.addOverride(file.dossierId(), file.fileId(), overrideEntry); + }); + + if (file.softDeleted != null) { + componentLogService.softDeleteAllOverrides(file.dossierId(), file.fileId()); + } + + fileManagementStorageService.deleteObject(file.dossierId(), file.fileId(), FileType.COMPONENTS); + } + + + private Optional getComponentOverridesFromStorageForMigration(String dossierId, String fileId) { + + try { + return Optional.ofNullable(storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.COMPONENTS), + ComponentsOverrides.class)); + } catch (StorageObjectDoesNotExist e) { + log.debug("Component overrides do not exist"); + } catch (StorageException e) { + log.debug(e.getMessage()); + } + return Optional.empty(); + } + + + private ComponentLogEntry buildOverride(String componentName, String componentValue) { + + return ComponentLogEntry.builder() + .name(componentName) + .componentValues(List.of(ComponentLogEntryValue.builder() + .value(componentValue) + .valueDescription("no description") + .componentRuleId("OLD.0.0") + .componentLogEntityReferences(Collections.emptyList()) + .build())) + .overridden(true) + .build(); + } + + + public record DossierFile(String dossierId, String fileId, OffsetDateTime softDeleted, OffsetDateTime hardDeleted) { } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ComponentOverridesMigration21.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ComponentOverridesMigration21.java new file mode 100644 index 000000000..ce2e5291b --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ComponentOverridesMigration21.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; +import com.iqser.red.service.persistence.management.v1.processor.migration.StorageToMongoCopyService; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class ComponentOverridesMigration21 extends Migration { + + private static final String NAME = "Migrate component overrides to mongoDB"; + private static final long VERSION = 21; + + @Autowired + StorageToMongoCopyService storageToMongoCopyService; + + + public ComponentOverridesMigration21() { + + super(NAME, VERSION); + } + + + @Override + protected void migrate() { + + storageToMongoCopyService.copyComponentOverrides(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java index 2eb674d17..0cb692751 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java @@ -55,7 +55,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration { } log.info("Migration: Copying all files for all dossiers to mongodb"); - storageToMongoCopyService.copy(); + storageToMongoCopyService.copyEntityLogs(); log.info("Migration: Finished Mongo db migration"); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration18.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration18.java index c03ff849e..2281d2a3c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration18.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration18.java @@ -30,7 +30,7 @@ public class StorageToMongoMigration18 extends Migration { protected void migrate() { log.info("Migration: Copying all files for all dossiers to mongodb"); - storageToMongoCopyService.copy(); + storageToMongoCopyService.copyEntityLogs(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentLogService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentLogService.java index 4dc459296..3d31b0cf8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentLogService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentLogService.java @@ -25,6 +25,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ComponentDocument; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.mapper.ComponentLogDocumentMapper; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; @@ -44,15 +46,16 @@ public class ComponentLogService { private final ComponentDefinitionPersistenceService componentDefinitionPersistenceService; private final UserService userService; private final DossierRepository dossierRepository; + private final ComponentLogDocumentMapper mapper = ComponentLogDocumentMapper.INSTANCE; public ComponentLog getComponentLog(String dossierId, String fileId, boolean includeOverrides) { - List orderedEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierRepository.findDossierTemplateId(dossierId)) + List orderedEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierRepository.findDossierTemplateId( + dossierId)) .stream() .sorted(Comparator.comparing(ComponentDefinitionEntity::getRank)) .toList(); - orderedEntities.forEach(o -> log.info("Name: {} Rank: {}", o.getTechnicalName(), o.getRank())); List orderedNames = orderedEntities.stream() .map(ComponentDefinitionEntity::getTechnicalName) .collect(Collectors.toList()); @@ -143,51 +146,59 @@ public class ComponentLogService { @Transactional - public void addOverride(String dossierId, String fileId, ComponentLogEntry componentOverride) { + public ComponentLogEntry addOverride(String dossierId, String fileId, ComponentLogEntry componentOverride) { var optionalComponentLogEntry = componentLogMongoService.findComponentLogEntryById(dossierId, fileId, componentOverride.getName()); ComponentLogEntry componentToUpdate; + ComponentDocument result; if (optionalComponentLogEntry.isPresent()) { componentToUpdate = optionalComponentLogEntry.get(); - updateComponentLogEntry(dossierId, fileId, componentOverride, componentToUpdate); + result = updateComponentLogEntry(dossierId, fileId, componentOverride, componentToUpdate); } else { - insertOverride(dossierId, fileId, componentOverride); + result = insertOverride(dossierId, fileId, componentOverride); auditOverride(dossierId, fileId, componentOverride); } + return mapper.fromComponentDocument(result); } - private void updateComponentLogEntry(String dossierId, String fileId, ComponentLogEntry componentOverride, ComponentLogEntry componentToUpdate) { + private ComponentDocument updateComponentLogEntry(String dossierId, String fileId, ComponentLogEntry componentOverride, ComponentLogEntry componentToUpdate) { componentToUpdate.setComponentValues(componentOverride.getComponentValues()); - saveOverride(dossierId, fileId, componentToUpdate); + ComponentDocument overrides = saveOverride(dossierId, fileId, componentToUpdate); auditOverride(dossierId, fileId, componentToUpdate); + return overrides; } - private void saveOverride(String dossierId, String fileId, ComponentLogEntry componentToUpdate) { - - componentLogMongoService.saveComponentLogEntries(dossierId, fileId, List.of(componentToUpdate)); + private ComponentDocument saveOverride(String dossierId, String fileId, ComponentLogEntry componentToUpdate) { + List overrides = componentLogMongoService.saveComponentLogEntries(dossierId, fileId, List.of(componentToUpdate)); + assert overrides.size() == 1; + return overrides.get(0); } + private void hardDeleteAllOverrides(String dossierId, String fileId) { componentLogMongoService.hardDeleteComponentLogEntries(dossierId, fileId); } - private void softDeleteAllOverrides(String dossierId, String fileId) { + + public void softDeleteAllOverrides(String dossierId, String fileId) { componentLogMongoService.softDeleteComponentLogEntries(dossierId, fileId); } - private void insertOverride(String dossierId, String fileId, ComponentLogEntry componentToAdd) { + private ComponentDocument insertOverride(String dossierId, String fileId, ComponentLogEntry componentToAdd) { - componentLogMongoService.insertComponentLogEntries(dossierId, fileId, List.of(componentToAdd)); + List overrides = componentLogMongoService.insertComponentLogEntries(dossierId, fileId, List.of(componentToAdd)); + assert overrides.size() == 1; + return overrides.get(0); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/StorageToMongoDBPerformanceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/StorageToMongoDBPerformanceTest.java index 0db90f601..60b1c6fbd 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/StorageToMongoDBPerformanceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/StorageToMongoDBPerformanceTest.java @@ -1,7 +1,6 @@ package com.iqser.red.service.peristence.v1.server.integration.tests.performance; import java.io.FileInputStream; -import java.io.IOException; import java.util.LinkedList; import java.util.concurrent.CompletableFuture; @@ -63,7 +62,7 @@ public class StorageToMongoDBPerformanceTest extends AbstractPersistenceServerSe futures.forEach(CompletableFuture::join); TenantContext.setTenantId(TENANT_1); long start = System.currentTimeMillis(); - storageToMongoCopyService.copy(); + storageToMongoCopyService.copyEntityLogs(); long end = System.currentTimeMillis(); System.out.println("Saving of" + NUMBER_OF_FILES + " files took " + (end - start) + "ms"); } diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ComponentLogMongoService.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ComponentLogMongoService.java index 6b9578643..28b2c3198 100644 --- a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ComponentLogMongoService.java +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ComponentLogMongoService.java @@ -93,7 +93,7 @@ public class ComponentLogMongoService { } - public void insertComponentLogEntries(String dossierId, String fileId, List componentLogEntries) { + public List insertComponentLogEntries(String dossierId, String fileId, List componentLogEntries) { String componentLogId = mapper.getComponentLogId(dossierId, fileId); @@ -101,7 +101,7 @@ public class ComponentLogMongoService { .map(componentLogEntry -> mapper.toComponentDocument(componentLogId, componentLogEntry)) .toList(); - componentDocumentRepository.insert(componentDocuments); + return componentDocumentRepository.insert(componentDocuments); } @@ -137,7 +137,7 @@ public class ComponentLogMongoService { } - public void saveComponentLogEntries(String dossierId, String fileId, List componentLogEntries) { + public List saveComponentLogEntries(String dossierId, String fileId, List componentLogEntries) { String componentLogId = mapper.getComponentLogId(dossierId, fileId); @@ -145,7 +145,7 @@ public class ComponentLogMongoService { .map(componentLogEntry -> mapper.toComponentDocument(componentLogId, componentLogEntry)) .toList(); - componentDocumentRepository.saveAll(componentDocuments); + return componentDocumentRepository.saveAll(componentDocuments); }