Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source
This commit is contained in:
maverickstuder 2024-03-19 11:26:52 +01:00
parent cfdd6a6924
commit 3060f5f28a
12 changed files with 145 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -45,6 +45,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE;
import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType.ENTITY_LOG;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
@Slf4j
@ -203,7 +204,7 @@ public class SaasMigrationService implements TenantSyncService {
//todo: 8702
private boolean entityLogMigrationFilesExist(String dossierId, String fileId) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists(
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, ENTITY_LOG)) && storageService.objectExists(
TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS));
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -12,7 +12,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntityLogMongoService;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntityLogMongoService.java → persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/EntityLogMongoService.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
package com.iqser.red.service.persistence.management.v1.processor.service.mongo;
import java.util.ArrayList;
import java.util.HashSet;

17
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogDocumentRepository.java Normal file
View File

@ -0,0 +1,17 @@
package com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository;
import java.util.Optional;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
@Repository
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {
@Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }")
Optional<EntityLogDocument> findAnalysisNumberById(@Param("_id") String id);
}

25
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogEntryDocumentRepository.java Normal file
View File

@ -0,0 +1,25 @@
package com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository;
import java.util.List;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
@Repository
public interface EntityLogEntryDocumentRepository extends MongoRepository<EntityLogEntryDocument, String> {
@Query("{ 'entityLogId' : ?0, 'manualChanges' : { $exists: true, $not: { $size: 0 } } }")
List<EntityLogEntryDocument> findByEntityLogIdAndManualChangesNotEmpty(String entityLogId);
@Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : ?1 }")
List<EntityLogEntryDocument> findByEntityLogIdAndChangesAnalysisNumber(String entityLogId, int analysisNumber);
@Query("{ 'entityLogId' : ?0}")
List<EntityLogEntryDocument> findByEntityLogId(String entityLogId);
@Query(value = "{ 'entityLogId' : ?0}", delete = true)
void deleteByEntityLogId(String entityLogId);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogDocumentRepository.java
View File

@ -5,9 +5,11 @@ import java.util.Optional;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
@Repository
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {
@Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }")

1
persistence-service-v1/persistence-service-server-v1/build.gradle.kts
View File

@ -23,6 +23,7 @@ dependencies {
api(project(":persistence-service-external-api-impl-v2"))
api(project(":persistence-service-internal-api-impl-v1"))
api("com.iqser.red.commons:storage-commons:2.45.0")
api("com.knecon.fforesight:mongo-database-commons:maverick-mongo")
api("junit:junit:4.13.2")
api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
api("net.logstash.logback:logstash-logback-encoder:7.4")

9
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
View File

@ -14,6 +14,7 @@ import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfigurati
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cache.annotation.EnableCaching;
@ -21,6 +22,7 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import org.springframework.http.server.observation.DefaultServerRequestObservationConvention;
import org.springframework.http.server.observation.ServerRequestObservationContext;
import org.springframework.http.server.observation.ServerRequestObservationConvention;
@ -43,6 +45,8 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.databasetenantcommons.DatabaseTenantCommonsAutoConfiguration;
import com.knecon.fforesight.jobscommons.JobsAutoConfiguration;
import com.knecon.fforesight.keycloakcommons.DefaultKeyCloakCommonsAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.service.MongoDataSources;
import com.knecon.fforesight.swaggercommons.SpringDocAutoConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyMessagingConfiguration;
@ -62,9 +66,10 @@ import lombok.extern.slf4j.Slf4j;
@EnableScheduling
@EnableCaching
@EnableConfigurationProperties({FileManagementServiceSettings.class})
@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class})
@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, CassandraAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class})
@Import({PersistenceServiceExternalApiConfigurationV2.class, PersistenceServiceExternalApiConfiguration.class, PersistenceServiceInternalApiConfiguration.class, PersistenceServiceExternalApiCacheConfiguration.class, MultiTenancyWebConfiguration.class, PersistenceServiceProcessorConfiguration.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class})
@Import({PersistenceServiceExternalApiConfigurationV2.class, PersistenceServiceExternalApiConfiguration.class, PersistenceServiceInternalApiConfiguration.class, PersistenceServiceExternalApiCacheConfiguration.class, MultiTenancyWebConfiguration.class, PersistenceServiceProcessorConfiguration.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class, MongoDataSources.class})
@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository")
public class Application implements ApplicationContextAware {
/**

9
persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml
View File

@ -37,6 +37,15 @@ spring:
batch_size: 1000
order_inserts: true
order_updates: true
data:
mongodb:
auto-index-creation: true
# todo: multi-tenancy
database: redaction
host: ${MONGODB_HOST:localhost}
port: 27017
username: ${MONGODB_USER}
password: ${MONGODB_PASSWORD}
cache:
type: redis
mvc:

36
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.roles.Applicati
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
@ -28,6 +29,8 @@ import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.databasetenantcommons.providers.TenantCreatedListener;
import com.knecon.fforesight.databasetenantcommons.providers.events.TenantCreatedEvent;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.mongo.database.commons.config.MongoDbConfiguration;
import com.knecon.fforesight.mongo.database.commons.service.MongoDataSources;
import com.knecon.fforesight.tenantcommons.EncryptionDecryptionService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
@ -46,7 +49,9 @@ import org.springframework.amqp.core.AmqpAdmin;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
@ -54,6 +59,8 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.*;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.StatementCallback;
import org.springframework.jdbc.datasource.SingleConnectionDataSource;
@ -100,6 +107,12 @@ public abstract class AbstractPersistenceServerServiceTest {
@Autowired
protected StorageService storageService;
@Autowired
protected EntityLogDocumentRepository entityLogDocumentRepository;
@Autowired
protected EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
@Autowired
protected EntityLogMongoService entityLogMongoService;
@Autowired
protected FileManagementStorageService fileManagementStorageService;
@Autowired
protected DossierTemplateRepository dossierTemplateRepository;
@ -268,6 +281,7 @@ public abstract class AbstractPersistenceServerServiceTest {
private void createDefaultTenant() {
var postgreSQLContainerMaster = SpringPostgreSQLTestContainer.getInstance().withDatabaseName("integration-tests-db-master").withUsername("sa").withPassword("sa");
var mongoDbContainer = MongoDBTestContainer.getInstance();
var port = postgreSQLContainerMaster.getJdbcUrl().substring(0, postgreSQLContainerMaster.getJdbcUrl().lastIndexOf('/')).split(":")[3];
@ -296,6 +310,7 @@ public abstract class AbstractPersistenceServerServiceTest {
.numberOfShards("1")
.numberOfReplicas("5")
.build());
redactionTenant.setS3StorageConnection(S3StorageConnection.builder()
.key("key")
.secret("secret")
@ -304,6 +319,15 @@ public abstract class AbstractPersistenceServerServiceTest {
.region("eu")
.endpoint("endpoint")
.build());
redactionTenant.setMongoDBConnection(MongoDBConnection.builder()
.host(mongoDbContainer.getHost())
.port(String.valueOf(mongoDbContainer.getFirstMappedPort()))
.username(MongoDBTestContainer.MONGO_USERNAME)
.password(MongoDBTestContainer.MONGO_PASSWORD)
.database(MongoDBTestContainer.MONGO_DATABASE)
.build());
when(tenantsClient.getTenant("redaction")).thenReturn(redactionTenant);
when(tenantsClient.getTenants()).thenReturn(List.of(redactionTenant));
@ -385,6 +409,9 @@ public abstract class AbstractPersistenceServerServiceTest {
indexInformationRepository.deleteAll();
applicationConfigRepository.deleteAll();
entityLogEntryDocumentRepository.deleteAll();
entityLogDocumentRepository.deleteAll();
});
}
@ -401,10 +428,17 @@ public abstract class AbstractPersistenceServerServiceTest {
var redisContainer = RedisTestContainer.getInstance();
redisContainer.start();
var mongoInstance = MongoDBTestContainer.getInstance();
mongoInstance.start();
log.info("Hosts are - Redis: {}, Postgres: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost());
TestPropertyValues.of("REDIS_PORT=" + redisContainer.getFirstMappedPort(),
"REDIS_HOST=" + redisContainer.getHost(),
"MONGODB_HOST=" + mongoInstance.getHost(),
"MONGODB_PORT=" + mongoInstance.getFirstMappedPort(),
"MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME,
"MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD,
"fforesight.jobs.enabled=false",
"fforesight.keycloak.enabled=false").applyTo(configurableApplicationContext.getEnvironment());
@ -416,7 +450,9 @@ public abstract class AbstractPersistenceServerServiceTest {
@EnableWebSecurity
@EnableMethodSecurity
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({MongoDataSources.class, MongoDbConfiguration.class})
@ComponentScan("com.iqser.red.service.persistence")
@EnableMongoRepositories("com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository")
public static class TestConfiguration {
@Bean

35
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/MongoDBTestContainer.java Normal file
View File

@ -0,0 +1,35 @@
package com.iqser.red.service.peristence.v1.server.integration.utils;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.DockerImageName;
public final class MongoDBTestContainer extends GenericContainer<MongoDBTestContainer> {
private static final String IMAGE_VERSION = "mongo:7.0.2";
public static final Integer MONGO_PORT = 27017;
public static final String MONGO_DATABASE = "mongo_database";
public static final String MONGO_PASSWORD = "mongo_password";
public static final String MONGO_USERNAME = "mongo_username";
private static MongoDBTestContainer mongoDB;
private MongoDBTestContainer() {
super(DockerImageName.parse(IMAGE_VERSION));
}
public static MongoDBTestContainer getInstance() {
if (mongoDB == null) {
mongoDB = new MongoDBTestContainer().withEnv("MONGO_INITDB_ROOT_USERNAME", MONGO_USERNAME)
.withEnv("MONGO_INITDB_ROOT_PASSWORD", MONGO_PASSWORD)
.withEnv("MONGO_INITDB_DATABASE", MONGO_DATABASE)
.withExposedPorts(MONGO_PORT);
}
return mongoDB;
}
}

10
persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml
View File

@ -19,7 +19,15 @@ spring:
order_inserts: true
order_updates: true
open-in-view: true
data:
mongodb:
auto-index-creation: true
# todo: multi-tenancy
database: redaction
host: ${MONGODB_HOST:localhost}
port: 27017
username: ${MONGODB_USER}
password: ${MONGODB_PASSWORD}
rabbitmq:
host: ${RABBITMQ_HOST:localhost}
port: ${RABBITMQ_PORT:5672}