Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9272: increase storageToMongoDb performance #615

Merged
kilian.schuettler1 merged 1 commits from RED-9272-fp into master 2024-07-23 00:10:30 +02:00
Conversation 5 Commits 1 Files Changed 3 +93 -2
3 changed files with 93 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/StorageToMongoCopyService.java
View File

@ -49,8 +49,9 @@ public class StorageToMongoCopyService {
log.info("Reading dossier {} file {} from storage", dossierFile.dossierId, dossierFile.fileId);
Optional<EntityLog> entityLogFromStorage = getEntityLogFromStorageForMigration(dossierFile.dossierId, dossierFile.fileId);
if (entityLogFromStorage.isPresent()) {
log.info("File found, now saving in mongodb");
fileManagementStorageService.saveEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLogFromStorage.get());
log.info("File found, now saving in mongodb");
fileManagementStorageService.deleteEntityLog(dossierFile.dossierId, dossierFile.fileId);
fileManagementStorageService.insertEntityLog(dossierFile.dossierId, dossierFile.fileId, entityLogFromStorage.get());
log.info("Deleting old file from storage");
fileManagementStorageService.deleteObject(dossierFile.dossierId, dossierFile.fileId, FileType.ENTITY_LOG);
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -145,6 +145,12 @@ public class FileManagementStorageService {
entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog);
}
@SneakyThrows
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.insertEntityLog(dossierId, fileId, entityLog);
}
@SneakyThrows
public void saveComponentLog(String dossierId, String fileId, ComponentLog componentLog) {

84
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/StorageToMongoDBPerformanceTest.java Normal file
View File

@ -0,0 +1,84 @@
package com.iqser.red.service.peristence.v1.server.integration.tests.performance;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.LinkedList;
import java.util.concurrent.CompletableFuture;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.migration.StorageToMongoCopyService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@Disabled
public class StorageToMongoDBPerformanceTest extends AbstractPersistenceServerServiceTest {
public static final int NUMBER_OF_FILES = 10;
@Autowired
private StorageToMongoCopyService storageToMongoCopyService;
@Autowired
private StorageService storageService;
@Autowired
private FileTesterAndProvider fileTesterAndProvider;
@Autowired
private DossierTesterAndProvider dossierTesterAndProvider;
@Autowired
private EntityLogMongoService entityLogMongoService;
@BeforeEach
@SneakyThrows
public void setUp() {
}
@Test
@SneakyThrows
public void testSavePerformance() {
var dossier = dossierTesterAndProvider.provideTestDossier();
LinkedList<CompletableFuture<Void>> futures = new LinkedList<>();
for (int i = 0; i < NUMBER_OF_FILES; i++) {
int number = i;
futures.add(CompletableFuture.supplyAsync(() -> {
prepStorageForFile(dossier, number);
return null;
}));
}
futures.forEach(CompletableFuture::join);
TenantContext.setTenantId(TENANT_1);
long start = System.currentTimeMillis();
storageToMongoCopyService.copy();
long end = System.currentTimeMillis();
System.out.println("Saving of" + NUMBER_OF_FILES + " files took " + (end - start) + "ms");
}
@SneakyThrows
private void prepStorageForFile(Dossier dossier, int i) {
TenantContext.setTenantId(TENANT_1);
var file = fileTesterAndProvider.testAndProvideFile(dossier, "file" + i);
try (var in = new FileInputStream("/home/kschuettler/Dokumente/TestFiles/entitylogs/largeEntitylog.json")) {
storageService.storeObject(TENANT_1, StorageIdUtils.getStorageId(dossier.getId(), file.getId(), FileType.ENTITY_LOG), in);
}
entityLogMongoService.deleteEntityLog(dossier.getId(), file.getId());
}
}