From e7e9d8823848c5bcec2cff08b8496e19d141d0d5 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Tue, 26 Mar 2024 14:13:31 +0100 Subject: [PATCH] RED-8702: Explore document databases to store entityLog * removed observed * added migration --- .../migration/StorageToMongoCopyService.java | 80 +++++++++++++++++++ .../migrations/StorageToMongoMigration17.java | 37 +++++++++ .../service/FileManagementStorageService.java | 8 +- 3 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration17.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java new file mode 100644 index 000000000..ad948611e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java @@ -0,0 +1,80 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.storage.commons.exception.StorageException; +import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; +import com.iqser.red.storage.commons.service.StorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class StorageToMongoCopyService { + + private final FileManagementStorageService fileManagementStorageService; + private final StorageService storageService; + + private final DossierRepository dossierRepository; + private final FileRepository fileRepository; + + + @SneakyThrows + public void copy() { + + List dossierEntities = dossierRepository.findAll(); + List files = new ArrayList<>(); + dossierEntities.forEach(dossierEntity -> { + List byDossierId = fileRepository.findByDossierId(dossierEntity.getId()); + byDossierId.forEach(file -> files.add(new DossierFile(dossierEntity.getId(), file.getId()))); + }); + + for (DossierFile dossierFile : files) { + log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId); + Optional entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId); + if (entityLogFromStorage.isPresent()) { + log.info("File found, now saving in mongodb"); + fileManagementStorageService.saveEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLogFromStorage.get()); + log.info("Deleting old file from storage"); + fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG); + } + } + } + + + private Optional getEntityLogFromStorageForMigration(String dossierId, String fileId) { + + try { + return Optional.ofNullable(storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), + EntityLog.class)); + } catch (StorageObjectDoesNotExist e) { + log.debug("EntityLog does not exist"); + } catch (StorageException e) { + log.debug(e.getMessage()); + } + return Optional.empty(); + } + + + public record DossierFile(String dossierId, String fileId) { + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration17.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration17.java new file mode 100644 index 000000000..3fc03b85d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration17.java @@ -0,0 +1,37 @@ +package com.iqser.red.service.persistence.management.v1.processor.migration.migrations; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; +import com.iqser.red.service.persistence.management.v1.processor.migration.StorageToMongoCopyService; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Setter +@Service +public class StorageToMongoMigration17 extends Migration { + + private final StorageToMongoCopyService storageToMongoCopyService; + + private static final String NAME = "Migration for entity log storage from s3 to mongodb"; + private static final long VERSION = 17; + + + public StorageToMongoMigration17(StorageToMongoCopyService storageToMongoCopyService) { + + super(NAME, VERSION); + this.storageToMongoCopyService = storageToMongoCopyService; + } + + + @Override + protected void migrate() { + + log.info("Migration: Copying all files for all dossiers to mongodb"); + storageToMongoCopyService.copy(); + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java index 4ce1b0764..b315ee8be 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java @@ -25,7 +25,6 @@ import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; -import io.micrometer.observation.annotation.Observed; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -101,7 +100,6 @@ public class FileManagementStorageService { } - @Observed(name = "FileManagementStorageService", contextualName = "get-entity-log") public EntityLog getEntityLog(String dossierId, String fileId) { return entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId) @@ -109,14 +107,14 @@ public class FileManagementStorageService { } + @SneakyThrows - @Observed(name = "FileManagementStorageService", contextualName = "save-entity-log") public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) { entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog); } - @Observed(name = "FileManagementStorageService", contextualName = "entity-log-exists") + public boolean entityLogExists(String dossierId, String fileId) { return entityLogMongoService.entityLogDocumentExists(dossierId, fileId); @@ -174,7 +172,7 @@ public class FileManagementStorageService { storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType)); } - @Observed(name = "FileManagementStorageService", contextualName = "delete-entity-log") + public void deleteEntityLog(String dossierId, String fileId) { entityLogMongoService.deleteEntityLog(dossierId, fileId);