Merge branch 'RED-9902' into 'release/2.465.x'
RED-9902: copy overrides to mongo See merge request redactmanager/persistence-service!678
This commit is contained in:
Kilian Schüttler
commit
a4736ea277
@ -1,23 +1,29 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentsOverrides;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ComponentDocument;
|
||||
import com.iqser.red.storage.commons.exception.StorageException;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
@ -37,30 +43,49 @@ public class StorageToMongoCopyService {
|
||||
|
||||
private final DossierRepository dossierRepository;
|
||||
private final FileRepository fileRepository;
|
||||
private final ComponentLogService componentLogService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void copy() {
|
||||
public void copyEntityLogs() {
|
||||
|
||||
forEachFile(this::copyEntityLogs);
|
||||
}
|
||||
|
||||
|
||||
public void copyComponentOverrides() {
|
||||
|
||||
forEachFile(this::copyComponentOverrides);
|
||||
}
|
||||
|
||||
|
||||
private void forEachFile(Consumer<DossierFile> consumer) {
|
||||
|
||||
List<DossierEntity> dossierEntities = dossierRepository.findAll();
|
||||
List<DossierFile> files = new ArrayList<>();
|
||||
dossierEntities.forEach(dossierEntity -> {
|
||||
List<FileEntity> byDossierId = fileRepository.findByDossierId(dossierEntity.getId());
|
||||
byDossierId.forEach(file -> files.add(new DossierFile(dossierEntity.getId(), file.getId())));
|
||||
byDossierId.forEach(file -> files.add(new DossierFile(dossierEntity.getId(), file.getId(), file.getDeleted(), file.getHardDeletedTime())));
|
||||
});
|
||||
|
||||
for (DossierFile dossierFile : files) {
|
||||
log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId);
|
||||
Optional<EntityLog> entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId);
|
||||
if (entityLogFromStorage.isPresent()) {
|
||||
EntityLog entityLog = removeDuplicates(entityLogFromStorage.get());
|
||||
log.info("File found, now saving in mongodb");
|
||||
fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId);
|
||||
fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLog);
|
||||
log.info("Deleting old file from storage");
|
||||
fileManagementStorageService.storeJSONObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG_BAK, entityLogFromStorage.get());
|
||||
fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG);
|
||||
}
|
||||
consumer.accept(dossierFile);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void copyEntityLogs(DossierFile dossierFile) {
|
||||
|
||||
log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId);
|
||||
Optional<EntityLog> entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId);
|
||||
if (entityLogFromStorage.isPresent()) {
|
||||
EntityLog entityLog = removeDuplicates(entityLogFromStorage.get());
|
||||
log.info("File found, now saving in mongodb");
|
||||
fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId);
|
||||
fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLog);
|
||||
log.info("Deleting old file from storage");
|
||||
fileManagementStorageService.storeJSONObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG_BAK, entityLogFromStorage.get());
|
||||
fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG);
|
||||
}
|
||||
}
|
||||
|
||||
@ -102,7 +127,62 @@ public class StorageToMongoCopyService {
|
||||
}
|
||||
|
||||
|
||||
public record DossierFile(String dossierId, String fileId) {
|
||||
private void copyComponentOverrides(DossierFile file) {
|
||||
|
||||
if (file.hardDeleted != null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Optional<ComponentsOverrides> optionalComponentsOverrides = getComponentOverridesFromStorageForMigration(file.dossierId(), file.fileId());
|
||||
if (optionalComponentsOverrides.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
ComponentsOverrides oldComponentsOverrides = optionalComponentsOverrides.get();
|
||||
oldComponentsOverrides.getComponentOverrides()
|
||||
.forEach((componentName, componentValue) -> {
|
||||
ComponentLogEntry overrideEntry = buildOverride(componentName, componentValue);
|
||||
ComponentLogEntry result = componentLogService.addOverride(file.dossierId(), file.fileId(), overrideEntry);
|
||||
});
|
||||
|
||||
if (file.softDeleted != null) {
|
||||
componentLogService.softDeleteAllOverrides(file.dossierId(), file.fileId());
|
||||
}
|
||||
|
||||
fileManagementStorageService.deleteObject(file.dossierId(), file.fileId(), FileType.COMPONENTS);
|
||||
}
|
||||
|
||||
|
||||
private Optional<ComponentsOverrides> getComponentOverridesFromStorageForMigration(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
return Optional.ofNullable(storageService.readJSONObject(TenantContext.getTenantId(),
|
||||
StorageIdUtils.getStorageId(dossierId, fileId, FileType.COMPONENTS),
|
||||
ComponentsOverrides.class));
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Component overrides do not exist");
|
||||
} catch (StorageException e) {
|
||||
log.debug(e.getMessage());
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
|
||||
private ComponentLogEntry buildOverride(String componentName, String componentValue) {
|
||||
|
||||
return ComponentLogEntry.builder()
|
||||
.name(componentName)
|
||||
.componentValues(List.of(ComponentLogEntryValue.builder()
|
||||
.value(componentValue)
|
||||
.valueDescription("no description")
|
||||
.componentRuleId("OLD.0.0")
|
||||
.componentLogEntityReferences(Collections.emptyList())
|
||||
.build()))
|
||||
.overridden(true)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
public record DossierFile(String dossierId, String fileId, OffsetDateTime softDeleted, OffsetDateTime hardDeleted) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,34 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.migration.StorageToMongoCopyService;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ComponentOverridesMigration21 extends Migration {
|
||||
|
||||
private static final String NAME = "Migrate component overrides to mongoDB";
|
||||
private static final long VERSION = 21;
|
||||
|
||||
@Autowired
|
||||
StorageToMongoCopyService storageToMongoCopyService;
|
||||
|
||||
|
||||
public ComponentOverridesMigration21() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
storageToMongoCopyService.copyComponentOverrides();
|
||||
}
|
||||
|
||||
}
|
||||
@ -55,7 +55,7 @@ public class DocumineLayoutRewriteMigration20 extends Migration {
|
||||
}
|
||||
|
||||
log.info("Migration: Copying all files for all dossiers to mongodb");
|
||||
storageToMongoCopyService.copy();
|
||||
storageToMongoCopyService.copyEntityLogs();
|
||||
log.info("Migration: Finished Mongo db migration");
|
||||
|
||||
|
||||
|
||||
@ -30,7 +30,7 @@ public class StorageToMongoMigration18 extends Migration {
|
||||
protected void migrate() {
|
||||
|
||||
log.info("Migration: Copying all files for all dossiers to mongodb");
|
||||
storageToMongoCopyService.copy();
|
||||
storageToMongoCopyService.copyEntityLogs();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -25,6 +25,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ComponentDocument;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.mapper.ComponentLogDocumentMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
|
||||
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
|
||||
|
||||
@ -44,15 +46,16 @@ public class ComponentLogService {
|
||||
private final ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
|
||||
private final UserService userService;
|
||||
private final DossierRepository dossierRepository;
|
||||
private final ComponentLogDocumentMapper mapper = ComponentLogDocumentMapper.INSTANCE;
|
||||
|
||||
|
||||
public ComponentLog getComponentLog(String dossierId, String fileId, boolean includeOverrides) {
|
||||
|
||||
List<ComponentDefinitionEntity> orderedEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierRepository.findDossierTemplateId(dossierId))
|
||||
List<ComponentDefinitionEntity> orderedEntities = componentDefinitionPersistenceService.findByDossierTemplateIdAndNotSoftDeleted(dossierRepository.findDossierTemplateId(
|
||||
dossierId))
|
||||
.stream()
|
||||
.sorted(Comparator.comparing(ComponentDefinitionEntity::getRank))
|
||||
.toList();
|
||||
orderedEntities.forEach(o -> log.info("Name: {} Rank: {}", o.getTechnicalName(), o.getRank()));
|
||||
List<String> orderedNames = orderedEntities.stream()
|
||||
.map(ComponentDefinitionEntity::getTechnicalName)
|
||||
.collect(Collectors.toList());
|
||||
@ -143,51 +146,59 @@ public class ComponentLogService {
|
||||
|
||||
|
||||
@Transactional
|
||||
public void addOverride(String dossierId, String fileId, ComponentLogEntry componentOverride) {
|
||||
public ComponentLogEntry addOverride(String dossierId, String fileId, ComponentLogEntry componentOverride) {
|
||||
|
||||
var optionalComponentLogEntry = componentLogMongoService.findComponentLogEntryById(dossierId, fileId, componentOverride.getName());
|
||||
|
||||
ComponentLogEntry componentToUpdate;
|
||||
ComponentDocument result;
|
||||
if (optionalComponentLogEntry.isPresent()) {
|
||||
componentToUpdate = optionalComponentLogEntry.get();
|
||||
updateComponentLogEntry(dossierId, fileId, componentOverride, componentToUpdate);
|
||||
result = updateComponentLogEntry(dossierId, fileId, componentOverride, componentToUpdate);
|
||||
} else {
|
||||
insertOverride(dossierId, fileId, componentOverride);
|
||||
result = insertOverride(dossierId, fileId, componentOverride);
|
||||
auditOverride(dossierId, fileId, componentOverride);
|
||||
}
|
||||
return mapper.fromComponentDocument(result);
|
||||
}
|
||||
|
||||
|
||||
private void updateComponentLogEntry(String dossierId, String fileId, ComponentLogEntry componentOverride, ComponentLogEntry componentToUpdate) {
|
||||
private ComponentDocument updateComponentLogEntry(String dossierId, String fileId, ComponentLogEntry componentOverride, ComponentLogEntry componentToUpdate) {
|
||||
|
||||
componentToUpdate.setComponentValues(componentOverride.getComponentValues());
|
||||
saveOverride(dossierId, fileId, componentToUpdate);
|
||||
ComponentDocument overrides = saveOverride(dossierId, fileId, componentToUpdate);
|
||||
auditOverride(dossierId, fileId, componentToUpdate);
|
||||
return overrides;
|
||||
}
|
||||
|
||||
|
||||
private void saveOverride(String dossierId, String fileId, ComponentLogEntry componentToUpdate) {
|
||||
|
||||
componentLogMongoService.saveComponentLogEntries(dossierId, fileId, List.of(componentToUpdate));
|
||||
private ComponentDocument saveOverride(String dossierId, String fileId, ComponentLogEntry componentToUpdate) {
|
||||
|
||||
List<ComponentDocument> overrides = componentLogMongoService.saveComponentLogEntries(dossierId, fileId, List.of(componentToUpdate));
|
||||
assert overrides.size() == 1;
|
||||
return overrides.get(0);
|
||||
}
|
||||
|
||||
|
||||
private void hardDeleteAllOverrides(String dossierId, String fileId) {
|
||||
|
||||
componentLogMongoService.hardDeleteComponentLogEntries(dossierId, fileId);
|
||||
|
||||
}
|
||||
|
||||
private void softDeleteAllOverrides(String dossierId, String fileId) {
|
||||
|
||||
public void softDeleteAllOverrides(String dossierId, String fileId) {
|
||||
|
||||
componentLogMongoService.softDeleteComponentLogEntries(dossierId, fileId);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void insertOverride(String dossierId, String fileId, ComponentLogEntry componentToAdd) {
|
||||
private ComponentDocument insertOverride(String dossierId, String fileId, ComponentLogEntry componentToAdd) {
|
||||
|
||||
componentLogMongoService.insertComponentLogEntries(dossierId, fileId, List.of(componentToAdd));
|
||||
List<ComponentDocument> overrides = componentLogMongoService.insertComponentLogEntries(dossierId, fileId, List.of(componentToAdd));
|
||||
assert overrides.size() == 1;
|
||||
return overrides.get(0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package com.iqser.red.service.peristence.v1.server.integration.tests.performance;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedList;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
@ -63,7 +62,7 @@ public class StorageToMongoDBPerformanceTest extends AbstractPersistenceServerSe
|
||||
futures.forEach(CompletableFuture::join);
|
||||
TenantContext.setTenantId(TENANT_1);
|
||||
long start = System.currentTimeMillis();
|
||||
storageToMongoCopyService.copy();
|
||||
storageToMongoCopyService.copyEntityLogs();
|
||||
long end = System.currentTimeMillis();
|
||||
System.out.println("Saving of" + NUMBER_OF_FILES + " files took " + (end - start) + "ms");
|
||||
}
|
||||
|
||||
@ -93,7 +93,7 @@ public class ComponentLogMongoService {
|
||||
}
|
||||
|
||||
|
||||
public void insertComponentLogEntries(String dossierId, String fileId, List<ComponentLogEntry> componentLogEntries) {
|
||||
public List<ComponentDocument> insertComponentLogEntries(String dossierId, String fileId, List<ComponentLogEntry> componentLogEntries) {
|
||||
|
||||
String componentLogId = mapper.getComponentLogId(dossierId, fileId);
|
||||
|
||||
@ -101,7 +101,7 @@ public class ComponentLogMongoService {
|
||||
.map(componentLogEntry -> mapper.toComponentDocument(componentLogId, componentLogEntry))
|
||||
.toList();
|
||||
|
||||
componentDocumentRepository.insert(componentDocuments);
|
||||
return componentDocumentRepository.insert(componentDocuments);
|
||||
}
|
||||
|
||||
|
||||
@ -137,7 +137,7 @@ public class ComponentLogMongoService {
|
||||
}
|
||||
|
||||
|
||||
public void saveComponentLogEntries(String dossierId, String fileId, List<ComponentLogEntry> componentLogEntries) {
|
||||
public List<ComponentDocument> saveComponentLogEntries(String dossierId, String fileId, List<ComponentLogEntry> componentLogEntries) {
|
||||
|
||||
String componentLogId = mapper.getComponentLogId(dossierId, fileId);
|
||||
|
||||
@ -145,7 +145,7 @@ public class ComponentLogMongoService {
|
||||
.map(componentLogEntry -> mapper.toComponentDocument(componentLogId, componentLogEntry))
|
||||
.toList();
|
||||
|
||||
componentDocumentRepository.saveAll(componentDocuments);
|
||||
return componentDocumentRepository.saveAll(componentDocuments);
|
||||
}
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user