Merge branch 'RED-9902' into 'release/2.465.x'

RED-9902: copy overrides to mongo

See merge request redactmanager/persistence-service!678

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java

@@ -1,23 +1,29 @@
 package com.iqser.red.service.persistence.management.v1.processor.migration;
 
+import java.time.OffsetDateTime;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
-import java.util.Set;
+import java.util.function.Consumer;
 
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
+import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
 import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentsOverrides;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ComponentDocument;
 import com.iqser.red.storage.commons.exception.StorageException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
@@ -37,30 +43,49 @@ public class StorageToMongoCopyService {
 
     private final DossierRepository dossierRepository;
     private final FileRepository fileRepository;
+    private final ComponentLogService componentLogService;
 
 
     @SneakyThrows
-    public void copy() {
+    public void copyEntityLogs() {
+
+        forEachFile(this::copyEntityLogs);
+    }
+
+
+    public void copyComponentOverrides() {
+
+        forEachFile(this::copyComponentOverrides);
+    }
+
+
+    private void forEachFile(Consumer<DossierFile> consumer) {
 
         List<DossierEntity> dossierEntities = dossierRepository.findAll();
         List<DossierFile> files = new ArrayList<>();
         dossierEntities.forEach(dossierEntity -> {
             List<FileEntity> byDossierId = fileRepository.findByDossierId(dossierEntity.getId());
-            byDossierId.forEach(file -> files.add(new DossierFile(dossierEntity.getId(), file.getId())));
+            byDossierId.forEach(file -> files.add(new DossierFile(dossierEntity.getId(), file.getId(), file.getDeleted(), file.getHardDeletedTime())));
         });
 
         for (DossierFile dossierFile : files) {
-            log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId);
+            consumer.accept(dossierFile);
-            Optional<EntityLog> entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId);
+        }
-            if (entityLogFromStorage.isPresent()) {
+    }
-                EntityLog entityLog = removeDuplicates(entityLogFromStorage.get());
+
-                log.info("File found, now saving in mongodb");
+
-                fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId);
+    private void copyEntityLogs(DossierFile dossierFile) {
-                fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLog);
+
-                log.info("Deleting old file from storage");
+        log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId);
-                fileManagementStorageService.storeJSONObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG_BAK, entityLogFromStorage.get());
+        Optional<EntityLog> entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId);
-                fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG);
+        if (entityLogFromStorage.isPresent()) {
-            }
+            EntityLog entityLog = removeDuplicates(entityLogFromStorage.get());
+            log.info("File found, now saving in mongodb");
+            fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId);
+            fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLog);
+            log.info("Deleting old file from storage");
+            fileManagementStorageService.storeJSONObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG_BAK, entityLogFromStorage.get());
+            fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG);
         }
     }
 
@@ -102,7 +127,62 @@ public class StorageToMongoCopyService {
     }
 
 
-    public record DossierFile(String dossierId, String fileId) {
+    private void copyComponentOverrides(DossierFile file) {
+
+        if (file.hardDeleted != null) {
+            return;
+        }
+
+        Optional<ComponentsOverrides> optionalComponentsOverrides = getComponentOverridesFromStorageForMigration(file.dossierId(), file.fileId());
+        if (optionalComponentsOverrides.isEmpty()) {
+            return;
+        }
+        ComponentsOverrides oldComponentsOverrides = optionalComponentsOverrides.get();
+        oldComponentsOverrides.getComponentOverrides()
+                .forEach((componentName, componentValue) -> {
+                    ComponentLogEntry overrideEntry = buildOverride(componentName, componentValue);
+                    ComponentLogEntry result = componentLogService.addOverride(file.dossierId(), file.fileId(), overrideEntry);
+                });
+
+        if (file.softDeleted != null) {
+            componentLogService.softDeleteAllOverrides(file.dossierId(), file.fileId());
+        }
+
+        fileManagementStorageService.deleteObject(file.dossierId(), file.fileId(), FileType.COMPONENTS);
+    }
+
+
+    private Optional<ComponentsOverrides> getComponentOverridesFromStorageForMigration(String dossierId, String fileId) {
+
+        try {
+            return Optional.ofNullable(storageService.readJSONObject(TenantContext.getTenantId(),
+                                                                     StorageIdUtils.getStorageId(dossierId, fileId, FileType.COMPONENTS),
+                                                                     ComponentsOverrides.class));
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Component overrides do not exist");
+        } catch (StorageException e) {
+            log.debug(e.getMessage());
+        }
+        return Optional.empty();
+    }
+
+
+    private ComponentLogEntry buildOverride(String componentName, String componentValue) {
+
+        return ComponentLogEntry.builder()
+                .name(componentName)
+                .componentValues(List.of(ComponentLogEntryValue.builder()
+                                                 .value(componentValue)
+                                                 .valueDescription("no description")
+                                                 .componentRuleId("OLD.0.0")
+                                                 .componentLogEntityReferences(Collections.emptyList())
+                                                 .build()))
+                .overridden(true)
+                .build();
+    }
+
+
+    public record DossierFile(String dossierId, String fileId, OffsetDateTime softDeleted, OffsetDateTime hardDeleted) {
 
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ComponentOverridesMigration21.java

@@ -0,0 +1,34 @@
+package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
+import com.iqser.red.service.persistence.management.v1.processor.migration.StorageToMongoCopyService;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class ComponentOverridesMigration21 extends Migration {
+
+    private static final String NAME = "Migrate component overrides to mongoDB";
+    private static final long VERSION = 21;
+
+    @Autowired
+    StorageToMongoCopyService storageToMongoCopyService;
+
+
+    public ComponentOverridesMigration21() {
+
+        super(NAME, VERSION);
+    }
+
+
+    @Override
+    protected void migrate() {
+
+        storageToMongoCopyService.copyComponentOverrides();
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DocumineLayoutRewriteMigration20.java

@@ -55,7 +55,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration  {
         }
 
         log.info("Migration: Copying all files for all dossiers to mongodb");
-        storageToMongoCopyService.copy();
+        storageToMongoCopyService.copyEntityLogs();
         log.info("Migration: Finished Mongo db migration");
 
         

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/StorageToMongoMigration18.java

@@ -30,7 +30,7 @@ public class StorageToMongoMigration18 extends Migration {
     protected void migrate() {
 
         log.info("Migration: Copying all files for all dossiers to mongodb");
-        storageToMongoCopyService.copy();
+        storageToMongoCopyService.copyEntityLogs();
 
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentLogService.java

@@ -25,6 +25,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ComponentDocument;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.mapper.ComponentLogDocumentMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
@@ -44,15 +46,16 @@ public class ComponentLogService {
     private final ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
     private final UserService userService;
     private final DossierRepository dossierRepository;
+    private final ComponentLogDocumentMapper mapper = ComponentLogDocumentMapper.INSTANCE;
 
 
     public ComponentLog getComponentLog(String dossierId, String fileId, boolean includeOverrides) {
 
-        List<ComponentDefinitionEntity> orderedEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierRepository.findDossierTemplateId(dossierId))
+        List<ComponentDefinitionEntity> orderedEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierRepository.findDossierTemplateId(
+                        dossierId))
                 .stream()
                 .sorted(Comparator.comparing(ComponentDefinitionEntity::getRank))
                 .toList();
-        orderedEntities.forEach(o -> log.info("Name: {} Rank: {}", o.getTechnicalName(), o.getRank()));
         List<String> orderedNames = orderedEntities.stream()
                 .map(ComponentDefinitionEntity::getTechnicalName)
                 .collect(Collectors.toList());
@@ -143,51 +146,59 @@ public class ComponentLogService {
 
 
     @Transactional
-    public void addOverride(String dossierId, String fileId, ComponentLogEntry componentOverride) {
+    public ComponentLogEntry addOverride(String dossierId, String fileId, ComponentLogEntry componentOverride) {
 
         var optionalComponentLogEntry = componentLogMongoService.findComponentLogEntryById(dossierId, fileId, componentOverride.getName());
 
         ComponentLogEntry componentToUpdate;
+        ComponentDocument result;
         if (optionalComponentLogEntry.isPresent()) {
             componentToUpdate = optionalComponentLogEntry.get();
-            updateComponentLogEntry(dossierId, fileId, componentOverride, componentToUpdate);
+            result = updateComponentLogEntry(dossierId, fileId, componentOverride, componentToUpdate);
         } else {
-            insertOverride(dossierId, fileId, componentOverride);
+            result = insertOverride(dossierId, fileId, componentOverride);
             auditOverride(dossierId, fileId, componentOverride);
         }
+        return mapper.fromComponentDocument(result);
     }
 
 
-    private void updateComponentLogEntry(String dossierId, String fileId, ComponentLogEntry componentOverride, ComponentLogEntry componentToUpdate) {
+    private ComponentDocument updateComponentLogEntry(String dossierId, String fileId, ComponentLogEntry componentOverride, ComponentLogEntry componentToUpdate) {
 
         componentToUpdate.setComponentValues(componentOverride.getComponentValues());
-        saveOverride(dossierId, fileId, componentToUpdate);
+        ComponentDocument overrides = saveOverride(dossierId, fileId, componentToUpdate);
         auditOverride(dossierId, fileId, componentToUpdate);
+        return overrides;
     }
 
 
-    private void saveOverride(String dossierId, String fileId, ComponentLogEntry componentToUpdate) {
+    private ComponentDocument saveOverride(String dossierId, String fileId, ComponentLogEntry componentToUpdate) {
-
-        componentLogMongoService.saveComponentLogEntries(dossierId, fileId, List.of(componentToUpdate));
 
+        List<ComponentDocument> overrides = componentLogMongoService.saveComponentLogEntries(dossierId, fileId, List.of(componentToUpdate));
+        assert overrides.size() == 1;
+        return overrides.get(0);
     }
 
+
     private void hardDeleteAllOverrides(String dossierId, String fileId) {
 
         componentLogMongoService.hardDeleteComponentLogEntries(dossierId, fileId);
 
     }
 
-    private void softDeleteAllOverrides(String dossierId, String fileId) {
+
+    public void softDeleteAllOverrides(String dossierId, String fileId) {
 
         componentLogMongoService.softDeleteComponentLogEntries(dossierId, fileId);
 
     }
 
 
-    private void insertOverride(String dossierId, String fileId, ComponentLogEntry componentToAdd) {
+    private ComponentDocument insertOverride(String dossierId, String fileId, ComponentLogEntry componentToAdd) {
 
-        componentLogMongoService.insertComponentLogEntries(dossierId, fileId, List.of(componentToAdd));
+        List<ComponentDocument> overrides = componentLogMongoService.insertComponentLogEntries(dossierId, fileId, List.of(componentToAdd));
+        assert overrides.size() == 1;
+        return overrides.get(0);
     }
 
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/StorageToMongoDBPerformanceTest.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests.performance;
 
 import java.io.FileInputStream;
-import java.io.IOException;
 import java.util.LinkedList;
 import java.util.concurrent.CompletableFuture;
 
@@ -63,7 +62,7 @@ public class StorageToMongoDBPerformanceTest extends AbstractPersistenceServerSe
         futures.forEach(CompletableFuture::join);
         TenantContext.setTenantId(TENANT_1);
         long start = System.currentTimeMillis();
-        storageToMongoCopyService.copy();
+        storageToMongoCopyService.copyEntityLogs();
         long end = System.currentTimeMillis();
         System.out.println("Saving of" + NUMBER_OF_FILES + " files took " + (end - start) + "ms");
     }

persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ComponentLogMongoService.java

@@ -93,7 +93,7 @@ public class ComponentLogMongoService {
     }
 
 
-    public void insertComponentLogEntries(String dossierId, String fileId, List<ComponentLogEntry> componentLogEntries) {
+    public List<ComponentDocument> insertComponentLogEntries(String dossierId, String fileId, List<ComponentLogEntry> componentLogEntries) {
 
         String componentLogId = mapper.getComponentLogId(dossierId, fileId);
 
@@ -101,7 +101,7 @@ public class ComponentLogMongoService {
                 .map(componentLogEntry -> mapper.toComponentDocument(componentLogId, componentLogEntry))
                 .toList();
 
-        componentDocumentRepository.insert(componentDocuments);
+        return componentDocumentRepository.insert(componentDocuments);
     }
 
 
@@ -137,7 +137,7 @@ public class ComponentLogMongoService {
     }
 
 
-    public void saveComponentLogEntries(String dossierId, String fileId, List<ComponentLogEntry> componentLogEntries) {
+    public List<ComponentDocument> saveComponentLogEntries(String dossierId, String fileId, List<ComponentLogEntry> componentLogEntries) {
 
         String componentLogId = mapper.getComponentLogId(dossierId, fileId);
 
@@ -145,7 +145,7 @@ public class ComponentLogMongoService {
                 .map(componentLogEntry -> mapper.toComponentDocument(componentLogId, componentLogEntry))
                 .toList();
 
-        componentDocumentRepository.saveAll(componentDocuments);
+        return componentDocumentRepository.saveAll(componentDocuments);
     }