RED-8702: Explore document databases to store entityLog

This commit is contained in:
Maverick Studer 2024-04-03 12:48:43 +02:00
parent e821371806
commit 5bda827c21
12 changed files with 412 additions and 195 deletions

View File

@ -96,15 +96,15 @@ configurations {
dependencies {
implementation("com.knecon.fforesight:database-tenant-commons:0.21.0")
implementation("com.knecon.fforesight:keycloak-commons:0.25.0")
implementation("com.knecon.fforesight:database-tenant-commons:0.23.0")
implementation("com.knecon.fforesight:keycloak-commons:0.27.0")
implementation("com.knecon.fforesight:swagger-commons:0.7.0")
implementation("com.knecon.fforesight:tracing-commons:0.5.0")
implementation("net.logstash.logback:logstash-logback-encoder:7.4")
implementation("ch.qos.logback:logback-classic")
implementation("org.postgresql:postgresql:42.5.4")
implementation("com.google.guava:guava:33.0.0-jre")
implementation("org.liquibase:liquibase-core:4.17.2")
implementation("org.liquibase:liquibase-core:4.20.0")
implementation("org.keycloak:keycloak-admin-client:23.0.6")
implementation("org.springframework.boot:spring-boot-starter-amqp")
implementation("org.springframework.boot:spring-boot-starter-validation")
@ -114,6 +114,7 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-security")
implementation("org.springframework.boot:spring-boot-starter-web")
implementation("org.springframework.boot:spring-boot-starter-data-jpa")
implementation("org.springframework.boot:spring-boot-starter-data-mongodb")
implementation("org.apache.commons:commons-lang3:3.12.0")
implementation("commons-validator:commons-validator:1.8.0")
implementation("org.springframework.boot:spring-boot-configuration-processor")
@ -126,14 +127,14 @@ dependencies {
annotationProcessor("org.springframework.boot:spring-boot-configuration-processor")
testImplementation("org.springframework.boot:spring-boot-starter-test")
testImplementation("org.springframework.amqp:spring-rabbit-test")
testImplementation("org.testcontainers:postgresql:1.19.4")
testImplementation("org.testcontainers:testcontainers:1.19.4")
testImplementation("org.testcontainers:junit-jupiter:1.19.4")
testImplementation("org.testcontainers:postgresql:1.19.7")
testImplementation("org.testcontainers:testcontainers:1.19.7")
testImplementation("org.testcontainers:junit-jupiter:1.19.7")
testImplementation("com.github.dasniko:testcontainers-keycloak:3.2.0")
}
extra["springCloudVersion"] = "2022.0.2"
extra["testcontainersVersion"] = "1.17.6"
extra["springCloudVersion"] = "2022.0.4"
extra["testcontainersVersion"] = "1.19.7"
group = "com.knecon.fforesight"

View File

@ -4,7 +4,9 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration;
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import com.knecon.fforesight.keycloakcommons.DefaultKeyCloakCommonsAutoConfiguration;
@ -15,7 +17,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, LiquibaseAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class, SpringDocAutoConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, CassandraAutoConfiguration.class,})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, CassandraAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
public class TenantUserManagementServiceApplication {
/**

View File

@ -0,0 +1,28 @@
package com.knecon.fforesight.tenantusermanagement.entity;
import jakarta.persistence.Column;
import jakarta.persistence.Embeddable;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Embeddable
public class MongoDBConnectionEntity {
@Column(name = "mongodb_host")
private String host;
@Column(name = "mongodb_port")
private String port;
@Column(name = "mongodb_username")
private String username;
@Column(name = "mongodb_password")
private String password;
@Column(name = "mongodb_database")
private String database;
}

View File

@ -45,6 +45,9 @@ public class TenantEntity {
@Embedded
private S3StorageConnectionEntity s3StorageConnection;
@Embedded
private MongoDBConnectionEntity mongoDBConnection;
@Basic(fetch = FetchType.EAGER)
@Column(columnDefinition = "text")
@Convert(converter = JSONMapConverter.class)

View File

@ -6,7 +6,7 @@ import java.util.List;
import com.knecon.fforesight.tenantcommons.model.AzureStorageConnection;
import com.knecon.fforesight.tenantcommons.model.DatabaseConnection;
import com.knecon.fforesight.tenantcommons.model.S3StorageConnection;
import com.knecon.fforesight.tenantcommons.model.SearchConnection;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
@ -41,6 +41,8 @@ public class TenantRequest {
private AzureStorageConnection azureStorageConnection;
@Schema(description = "Parameter containing data of the S3 storage connection.")
private S3StorageConnection s3StorageConnection;
@Schema(description = "Parameter containing data of the MongoDB connection.")
private MongoDBConnection mongoDBConnection;
@Builder.Default
@Schema(description = "Parameter containing a list of users of the tenant.")

View File

@ -15,16 +15,16 @@ import java.util.stream.Collectors;
import javax.sql.DataSource;
import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest;
import jakarta.ws.rs.NotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.bson.BsonArray;
import org.bson.BsonDocument;
import org.bson.BsonString;
import org.keycloak.representations.idm.ClientRepresentation;
import org.keycloak.representations.idm.CredentialRepresentation;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.RoleRepresentation;
import org.keycloak.representations.idm.RolesRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
@ -46,12 +46,15 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.AzureStorageConnection;
import com.knecon.fforesight.tenantcommons.model.DatabaseConnection;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import com.knecon.fforesight.tenantcommons.model.S3StorageConnection;
import com.knecon.fforesight.tenantcommons.model.SearchConnection;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest;
import com.knecon.fforesight.tenantusermanagement.config.StorageConfiguration;
import com.knecon.fforesight.tenantusermanagement.entity.AzureStorageConnectionEntity;
import com.knecon.fforesight.tenantusermanagement.entity.DatabaseConnectionEntity;
import com.knecon.fforesight.tenantusermanagement.entity.MongoDBConnectionEntity;
import com.knecon.fforesight.tenantusermanagement.entity.S3StorageConnectionEntity;
import com.knecon.fforesight.tenantusermanagement.entity.SearchConnectionEntity;
import com.knecon.fforesight.tenantusermanagement.entity.TenantEntity;
@ -62,11 +65,15 @@ import com.knecon.fforesight.tenantusermanagement.model.TenantUser;
import com.knecon.fforesight.tenantusermanagement.properties.TenantUserManagementProperties;
import com.knecon.fforesight.tenantusermanagement.repository.TenantRepository;
import com.knecon.fforesight.tenantusermanagement.utils.JDBCUtils;
import com.mongodb.MongoCommandException;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoDatabase;
import jakarta.ws.rs.NotFoundException;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
@ -106,7 +113,11 @@ public class TenantManagementService implements TenantProvider {
updateMasterTheme(tenantUserManagementProperties.getLoginTheme());
updateMasterDisplayName(tenantUserManagementProperties.getApplicationName());
log.info("Tenants are: {}", tenantRepository.findAll().stream().map(TenantEntity::getTenantId).toList());
log.info("Tenants are: {}",
tenantRepository.findAll()
.stream()
.map(TenantEntity::getTenantId)
.toList());
log.info("Requested to create tenant for: {}", tenantRequest.getTenantId());
try {
@ -121,59 +132,69 @@ public class TenantManagementService implements TenantProvider {
.displayName(tenantRequest.getDisplayName())
.guid(UUID.randomUUID().toString())
.databaseConnection(DatabaseConnectionEntity.builder()
.driver(tenantRequest.getDatabaseConnection().getDriver())
.host(tenantRequest.getDatabaseConnection().getHost())
.port(tenantRequest.getDatabaseConnection().getPort())
.database(tenantRequest.getDatabaseConnection().getDatabase())
.schema(tenantRequest.getDatabaseConnection().getSchema())
.username(tenantRequest.getDatabaseConnection().getUsername())
.password(encryptionService.encrypt(tenantRequest.getDatabaseConnection().getPassword()))
.build())
.driver(tenantRequest.getDatabaseConnection().getDriver())
.host(tenantRequest.getDatabaseConnection().getHost())
.port(tenantRequest.getDatabaseConnection().getPort())
.database(tenantRequest.getDatabaseConnection().getDatabase())
.schema(tenantRequest.getDatabaseConnection().getSchema())
.username(tenantRequest.getDatabaseConnection().getUsername())
.password(encryptionService.encrypt(tenantRequest.getDatabaseConnection().getPassword()))
.build())
.searchConnection(SearchConnectionEntity.builder()
.hosts(tenantRequest.getSearchConnection().getHosts())
.port(tenantRequest.getSearchConnection().getPort())
.scheme(tenantRequest.getSearchConnection().getScheme())
.username(tenantRequest.getSearchConnection().getUsername())
.password(encryptionService.encrypt(tenantRequest.getSearchConnection().getPassword()))
.numberOfShards(tenantRequest.getSearchConnection().getNumberOfShards())
.numberOfReplicas(tenantRequest.getSearchConnection().getNumberOfReplicas())
.indexPrefix(buildIndexPrefix(tenantRequest.getTenantId()))
.build())
.hosts(tenantRequest.getSearchConnection().getHosts())
.port(tenantRequest.getSearchConnection().getPort())
.scheme(tenantRequest.getSearchConnection().getScheme())
.username(tenantRequest.getSearchConnection().getUsername())
.password(encryptionService.encrypt(tenantRequest.getSearchConnection().getPassword()))
.numberOfShards(tenantRequest.getSearchConnection().getNumberOfShards())
.numberOfReplicas(tenantRequest.getSearchConnection().getNumberOfReplicas())
.indexPrefix(buildIndexPrefix(tenantRequest.getTenantId()))
.build())
.mongoDBConnection(MongoDBConnectionEntity.builder()
.host(tenantRequest.getMongoDBConnection().getHost())
.port(tenantRequest.getMongoDBConnection().getPort())
.username(tenantRequest.getMongoDBConnection().getUsername())
.password(encryptionService.encrypt(tenantRequest.getMongoDBConnection().getPassword()))
.database(tenantRequest.getMongoDBConnection().getDatabase())
.build())
.build();
if (tenantRequest.getAzureStorageConnection() != null) {
testAzureConnection(tenantRequest.getAzureStorageConnection().getConnectionString(), tenantRequest.getAzureStorageConnection().getContainerName());
tenantEntity.setAzureStorageConnection(AzureStorageConnectionEntity.builder()
.connectionString(encryptionService.encrypt(tenantRequest.getAzureStorageConnection().getConnectionString()))
.containerName(tenantRequest.getAzureStorageConnection().getContainerName())
.build());
.connectionString(encryptionService.encrypt(tenantRequest.getAzureStorageConnection().getConnectionString()))
.containerName(tenantRequest.getAzureStorageConnection().getContainerName())
.build());
}
if (tenantRequest.getS3StorageConnection() != null) {
testS3Connection(tenantRequest.getS3StorageConnection());
tenantEntity.setS3StorageConnection(S3StorageConnectionEntity.builder()
.key(tenantRequest.getS3StorageConnection().getKey())
.secret(encryptionService.encrypt(tenantRequest.getS3StorageConnection().getSecret()))
.signerType(tenantRequest.getS3StorageConnection().getSignerType())
.bucketName(tenantRequest.getS3StorageConnection().getBucketName())
.region(tenantRequest.getS3StorageConnection().getRegion())
.endpoint(tenantRequest.getS3StorageConnection().getEndpoint())
.build());
.key(tenantRequest.getS3StorageConnection().getKey())
.secret(encryptionService.encrypt(tenantRequest.getS3StorageConnection().getSecret()))
.signerType(tenantRequest.getS3StorageConnection().getSignerType())
.bucketName(tenantRequest.getS3StorageConnection().getBucketName())
.region(tenantRequest.getS3StorageConnection().getRegion())
.endpoint(tenantRequest.getS3StorageConnection().getEndpoint())
.build());
}
createSchema(tenantRequest);
log.info("Created schema for tenant: {}", tenantRequest.getTenantId());
createMongoDBDatabase(tenantRequest);
log.info("Created mongodb database for tenant: {}", tenantRequest.getTenantId());
propagateTenantToKeyCloak(tenantRequest.getTenantId(), tenantRequest.getDefaultUsers());
log.info("Updated roles for tenant: {}", tenantRequest.getTenantId());
var saved = tenantPersistenceService.save(tenantEntity);
JsonNode node;
log.info("Persisted tenant: {}", tenantRequest.getTenantId());
TenantContext.setTenantId(tenantEntity.getTenantId());
@ -208,6 +229,7 @@ public class TenantManagementService implements TenantProvider {
TenantContext.clear();
log.info("Dispatched delete index message for tenant: {}", tenant.getTenantId());
log.info("Deleting schema for tenant: {}", tenant.getTenantId());
deleteSchema(tenant);
if (tenant.getAzureStorageConnection() != null) {
log.info("Deleting azure blob for tenant: {}", tenantId);
@ -227,32 +249,28 @@ public class TenantManagementService implements TenantProvider {
if (tenant.getS3StorageConnection() != null) {
var s3StorageConnectionTemplate = tenant.getS3StorageConnection();
com.iqser.red.storage.commons.model.S3StorageConnection s3StorageConnection =
new com.iqser.red.storage.commons.model.S3StorageConnection(s3StorageConnectionTemplate.getKey(),
encryptionService.decrypt(s3StorageConnectionTemplate.getSecret()),
s3StorageConnectionTemplate.getSignerType(),
s3StorageConnectionTemplate.getBucketName(),
s3StorageConnectionTemplate.getRegion(),
s3StorageConnectionTemplate.getEndpoint());
com.iqser.red.storage.commons.model.S3StorageConnection s3StorageConnection;
s3StorageConnection = new com.iqser.red.storage.commons.model.S3StorageConnection(s3StorageConnectionTemplate.getKey(),
encryptionService.decrypt(s3StorageConnectionTemplate.getSecret()),
s3StorageConnectionTemplate.getSignerType(),
s3StorageConnectionTemplate.getBucketName(),
s3StorageConnectionTemplate.getRegion(),
s3StorageConnectionTemplate.getEndpoint());
log.info("Deleting s3 bucket for tenant: {}", tenantId);
try (var client = storageConfiguration.getS3StorageService().initAmazonS3(s3StorageConnection)) {
String bucketName = s3StorageConnection.getBucketName();
ListObjectsRequest listObjects = ListObjectsRequest
.builder()
.bucket(bucketName)
.build();
ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(bucketName).build();
ListObjectsResponse objectList = client.listObjects(listObjects);
for (S3Object object : objectList.contents()) {
// Delete each object
client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(object.key()).build());
}
DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder()
.bucket(bucketName)
.expectedBucketOwner(tenantId)
.build();
DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder().bucket(bucketName).expectedBucketOwner(tenantId).build();
client.deleteBucket(deleteBucketRequest);
}
}
log.info("Deleting mongodb database for tenant: {}", tenant.getTenantId());
deleteMongoDBDatabase(tenant);
deleteRealm(tenantId);
tenantRepository.deleteById(tenant.getTenantId());
@ -317,8 +335,8 @@ public class TenantManagementService implements TenantProvider {
var jdbcUrl = JDBCUtils.buildJdbcUrl(tenantRequest.getDatabaseConnection());
try (Connection connection = DriverManager.getConnection(jdbcUrl,
tenantRequest.getDatabaseConnection().getUsername(),
tenantRequest.getDatabaseConnection().getPassword())) {
tenantRequest.getDatabaseConnection().getUsername(),
tenantRequest.getDatabaseConnection().getPassword())) {
DataSource tenantDataSource = new SingleConnectionDataSource(connection, false);
JdbcTemplate jdbcTemplate = new JdbcTemplate(tenantDataSource);
String createStatement = "CREATE SCHEMA IF NOT EXISTS \"" + tenantRequest.getDatabaseConnection().getSchema() + "\"";
@ -338,8 +356,8 @@ public class TenantManagementService implements TenantProvider {
log.info("Deleting schema for tenant: {}", tenant.getTenantId());
var jdbcUrl = JDBCUtils.buildJdbcUrl(tenant.getDatabaseConnection());
try (Connection connection = DriverManager.getConnection(jdbcUrl,
tenant.getDatabaseConnection().getUsername(),
this.encryptionService.decrypt(tenant.getDatabaseConnection().getPassword()))) {
tenant.getDatabaseConnection().getUsername(),
this.encryptionService.decrypt(tenant.getDatabaseConnection().getPassword()))) {
DataSource tenantDataSource = new SingleConnectionDataSource(connection, false);
JdbcTemplate jdbcTemplate = new JdbcTemplate(tenantDataSource);
String deleteStatement = "DROP SCHEMA IF EXISTS \"" + tenant.getDatabaseConnection().getSchema() + "\" CASCADE;";
@ -351,6 +369,49 @@ public class TenantManagementService implements TenantProvider {
}
private void createMongoDBDatabase(TenantRequest tenant) {
MongoDBConnection mongoDBConnection = tenant.getMongoDBConnection();
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
mongoDBConnection.getUsername(),
mongoDBConnection.getPassword(),
mongoDBConnection.getHost(),
mongoDBConnection.getPort()))) {
MongoDatabase database = mongoClient.getDatabase(mongoDBConnection.getDatabase());
BsonDocument createUserCommand = new BsonDocument();
createUserCommand.append("createUser", new BsonString(mongoDBConnection.getUsername()));
createUserCommand.append("pwd", new BsonString(mongoDBConnection.getPassword()));
BsonArray roles = new BsonArray();
roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(mongoDBConnection.getDatabase())));
createUserCommand.append("roles", roles);
try {
database.runCommand(createUserCommand);
} catch (MongoCommandException mongoCommandException) {
// ignore user already exists
if (mongoCommandException.getErrorCode() != 51003) {
throw mongoCommandException;
}
}
}
}
private void deleteMongoDBDatabase(TenantResponse tenant) {
MongoDBConnection mongoDBConnection = tenant.getMongoDBConnection();
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
mongoDBConnection.getUsername(),
mongoDBConnection.getPassword(),
mongoDBConnection.getHost(),
mongoDBConnection.getPort()))) {
mongoClient.getDatabase(mongoDBConnection.getDatabase()).drop();
}
}
public void createOrUpdateRealm(String tenantId, List<TenantUser> users) {
if (syncRealmIfExists(tenantId)) {
@ -369,7 +430,9 @@ public class TenantManagementService implements TenantProvider {
realm.setRoles(getRealmRoles());
if (users != null) {
realm.setUsers(users.stream().map(this::toUserRepresentation).toList());
realm.setUsers(users.stream()
.map(this::toUserRepresentation)
.toList());
}
keycloak.getAdminClient().realms().create(realm);
@ -401,12 +464,15 @@ public class TenantManagementService implements TenantProvider {
existingRealm.setAccessTokenLifespan(tenantUserManagementProperties.getAccessTokenLifeSpan());
existingRealm.setSsoSessionIdleTimeout(tenantUserManagementProperties.getSsoSessionIdleTimeout());
var clients = getRealmClients();
var relevantClientNames = clients.stream().map(c -> c.getClientId().toLowerCase(Locale.getDefault())).collect(Collectors.toSet());
var relevantClientNames = clients.stream()
.map(c -> c.getClientId().toLowerCase(Locale.getDefault()))
.collect(Collectors.toSet());
var existingClients = keycloak.getAdminClient().realm(tenantId).clients().findAll();
existingClients.forEach(ec -> {
if (relevantClientNames.contains(ec.getClientId().toLowerCase(Locale.getDefault()))) {
log.info("Removing client: {}", ec.getName());
keycloak.getAdminClient().realm(tenantId).clients().get(ec.getId()).remove();
keycloak.getAdminClient().realm(tenantId).clients()
.get(ec.getId()).remove();
}
});
@ -516,7 +582,9 @@ public class TenantManagementService implements TenantProvider {
private boolean tryToAccessRealm(String tenantId) {
try {
return keycloak.getAdminClient().realms().findAll().stream().anyMatch(r -> r.getRealm().equals(tenantId));
return keycloak.getAdminClient().realms().findAll()
.stream()
.anyMatch(r -> r.getRealm().equals(tenantId));
} catch (Exception e) {
return false;
}
@ -572,7 +640,9 @@ public class TenantManagementService implements TenantProvider {
public TenantResponse getTenant(String tenantId) {
return tenantRepository.findById(tenantId).map(this::convert).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Tenant does not exist"));
return tenantRepository.findById(tenantId)
.map(this::convert)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Tenant does not exist"));
}
@ -591,29 +661,29 @@ public class TenantManagementService implements TenantProvider {
var databaseConnection = tenantRequest.getDatabaseConnection();
if (databaseConnection != null) {
tenantEntity.setDatabaseConnection(DatabaseConnectionEntity.builder()
.driver(databaseConnection.getDriver())
.host(databaseConnection.getHost())
.port(databaseConnection.getPort())
.database(databaseConnection.getDatabase())
.schema(databaseConnection.getSchema())
.username(databaseConnection.getUsername())
.password(encryptionService.encrypt(databaseConnection.getPassword()))
.params(databaseConnection.getParams())
.build());
.driver(databaseConnection.getDriver())
.host(databaseConnection.getHost())
.port(databaseConnection.getPort())
.database(databaseConnection.getDatabase())
.schema(databaseConnection.getSchema())
.username(databaseConnection.getUsername())
.password(encryptionService.encrypt(databaseConnection.getPassword()))
.params(databaseConnection.getParams())
.build());
}
var searchConnection = tenantRequest.getSearchConnection();
if (searchConnection != null) {
tenantEntity.setSearchConnection(SearchConnectionEntity.builder()
.hosts(searchConnection.getHosts())
.port(searchConnection.getPort())
.scheme(searchConnection.getScheme())
.username(searchConnection.getUsername())
.password(encryptionService.encrypt(searchConnection.getPassword()))
.numberOfShards(searchConnection.getNumberOfShards())
.numberOfReplicas(searchConnection.getNumberOfReplicas())
.indexPrefix(tenantEntity.getSearchConnection().getIndexPrefix())
.build());
.hosts(searchConnection.getHosts())
.port(searchConnection.getPort())
.scheme(searchConnection.getScheme())
.username(searchConnection.getUsername())
.password(encryptionService.encrypt(searchConnection.getPassword()))
.numberOfShards(searchConnection.getNumberOfShards())
.numberOfReplicas(searchConnection.getNumberOfReplicas())
.indexPrefix(tenantEntity.getSearchConnection().getIndexPrefix())
.build());
}
var azureStorageConnection = tenantRequest.getAzureStorageConnection();
@ -623,9 +693,9 @@ public class TenantManagementService implements TenantProvider {
}
testAzureConnection(azureStorageConnection.getConnectionString(), azureStorageConnection.getContainerName());
tenantEntity.setAzureStorageConnection(AzureStorageConnectionEntity.builder()
.connectionString(encryptionService.encrypt(azureStorageConnection.getConnectionString()))
.containerName(azureStorageConnection.getContainerName())
.build());
.connectionString(encryptionService.encrypt(azureStorageConnection.getConnectionString()))
.containerName(azureStorageConnection.getContainerName())
.build());
} else {
tenantEntity.setAzureStorageConnection(null);
}
@ -637,17 +707,28 @@ public class TenantManagementService implements TenantProvider {
}
testS3Connection(s3StorageConnection);
tenantEntity.setS3StorageConnection(S3StorageConnectionEntity.builder()
.key(s3StorageConnection.getKey())
.secret(encryptionService.encrypt(s3StorageConnection.getSecret()))
.signerType(s3StorageConnection.getSignerType())
.bucketName(s3StorageConnection.getBucketName())
.region(s3StorageConnection.getRegion())
.endpoint(s3StorageConnection.getEndpoint())
.build());
.key(s3StorageConnection.getKey())
.secret(encryptionService.encrypt(s3StorageConnection.getSecret()))
.signerType(s3StorageConnection.getSignerType())
.bucketName(s3StorageConnection.getBucketName())
.region(s3StorageConnection.getRegion())
.endpoint(s3StorageConnection.getEndpoint())
.build());
} else {
tenantEntity.setS3StorageConnection(null);
}
var mongoDBConnection = tenantRequest.getMongoDBConnection();
if (mongoDBConnection != null) {
tenantEntity.setMongoDBConnection(MongoDBConnectionEntity.builder()
.host(mongoDBConnection.getHost())
.port(mongoDBConnection.getPort())
.username(mongoDBConnection.getUsername())
.password(encryptionService.encrypt(mongoDBConnection.getPassword()))
.database(mongoDBConnection.getDatabase())
.build());
}
return convert(tenantRepository.save(tenantEntity));
} else {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Tenant does not exist");
@ -657,7 +738,10 @@ public class TenantManagementService implements TenantProvider {
public List<TenantResponse> getTenants() {
return tenantRepository.findAll().stream().map(this::convert).toList();
return tenantRepository.findAll()
.stream()
.map(this::convert)
.toList();
}
@ -679,6 +763,10 @@ public class TenantManagementService implements TenantProvider {
tenantResponse.getS3StorageConnection().setSecret(PASSWORD);
}
if (tenantResponse.getMongoDBConnection() != null) {
tenantResponse.getMongoDBConnection().setPassword(PASSWORD);
}
return tenantResponse;
}
@ -697,43 +785,50 @@ public class TenantManagementService implements TenantProvider {
.authDetails(authDetails)
.details(entity.getDetails())
.databaseConnection(DatabaseConnection.builder()
.driver(entity.getDatabaseConnection().getDriver())
.host(entity.getDatabaseConnection().getHost())
.port(entity.getDatabaseConnection().getPort())
.database(entity.getDatabaseConnection().getDatabase())
.schema(entity.getDatabaseConnection().getSchema())
.username(entity.getDatabaseConnection().getUsername())
.params(entity.getDatabaseConnection().getParams())
.password(entity.getDatabaseConnection().getPassword())
.build())
.driver(entity.getDatabaseConnection().getDriver())
.host(entity.getDatabaseConnection().getHost())
.port(entity.getDatabaseConnection().getPort())
.database(entity.getDatabaseConnection().getDatabase())
.schema(entity.getDatabaseConnection().getSchema())
.username(entity.getDatabaseConnection().getUsername())
.params(entity.getDatabaseConnection().getParams())
.password(entity.getDatabaseConnection().getPassword())
.build())
.searchConnection(SearchConnection.builder()
.hosts(entity.getSearchConnection().getHosts())
.port(entity.getSearchConnection().getPort())
.scheme(entity.getSearchConnection().getScheme())
.username(entity.getSearchConnection().getUsername())
.numberOfShards(entity.getSearchConnection().getNumberOfShards())
.numberOfReplicas(entity.getSearchConnection().getNumberOfReplicas())
.password(entity.getSearchConnection().getPassword())
.indexPrefix(entity.getSearchConnection().getIndexPrefix())
.build())
.hosts(entity.getSearchConnection().getHosts())
.port(entity.getSearchConnection().getPort())
.scheme(entity.getSearchConnection().getScheme())
.username(entity.getSearchConnection().getUsername())
.numberOfShards(entity.getSearchConnection().getNumberOfShards())
.numberOfReplicas(entity.getSearchConnection().getNumberOfReplicas())
.password(entity.getSearchConnection().getPassword())
.indexPrefix(entity.getSearchConnection().getIndexPrefix())
.build())
.mongoDBConnection(MongoDBConnection.builder()
.host(entity.getMongoDBConnection().getHost())
.port(entity.getMongoDBConnection().getPort())
.username(entity.getMongoDBConnection().getUsername())
.password(entity.getMongoDBConnection().getPassword())
.database(entity.getMongoDBConnection().getDatabase())
.build())
.build();
if (entity.getAzureStorageConnection() != null) {
tenantResponse.setAzureStorageConnection(AzureStorageConnection.builder()
.connectionString(entity.getAzureStorageConnection().getConnectionString())
.containerName(entity.getAzureStorageConnection().getContainerName())
.build());
.connectionString(entity.getAzureStorageConnection().getConnectionString())
.containerName(entity.getAzureStorageConnection().getContainerName())
.build());
}
if (entity.getS3StorageConnection() != null) {
tenantResponse.setS3StorageConnection(S3StorageConnection.builder()
.key(entity.getS3StorageConnection().getKey())
.secret(entity.getS3StorageConnection().getSecret())
.signerType(entity.getS3StorageConnection().getSignerType())
.bucketName(entity.getS3StorageConnection().getBucketName())
.region(entity.getS3StorageConnection().getRegion())
.endpoint(entity.getS3StorageConnection().getEndpoint())
.build());
.key(entity.getS3StorageConnection().getKey())
.secret(entity.getS3StorageConnection().getSecret())
.signerType(entity.getS3StorageConnection().getSignerType())
.bucketName(entity.getS3StorageConnection().getBucketName())
.region(entity.getS3StorageConnection().getRegion())
.endpoint(entity.getS3StorageConnection().getEndpoint())
.build());
}
return tenantResponse;
@ -754,11 +849,11 @@ public class TenantManagementService implements TenantProvider {
var connection = storageConfiguration.getS3StorageService()
.testConnection(s3StorageConnection.getKey(),
s3StorageConnection.getSecret(),
s3StorageConnection.getSignerType(),
s3StorageConnection.getBucketName(),
s3StorageConnection.getRegion(),
s3StorageConnection.getEndpoint());
s3StorageConnection.getSecret(),
s3StorageConnection.getSignerType(),
s3StorageConnection.getBucketName(),
s3StorageConnection.getRegion(),
s3StorageConnection.getEndpoint());
if (!connection) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Could not connect to S3 storage");

View File

@ -13,3 +13,5 @@ databaseChangeLog:
file: db/changelog/master/7-change-index-name-to-index-prefix.changelog.yaml
- include:
file: db/changelog/master/8-tenant-connection-data-string-length.changelog.yaml
- include:
file: db/changelog/master/9-add-mongodb-connection-columns.yaml

View File

@ -0,0 +1,23 @@
databaseChangeLog:
- changeSet:
id: add-mongodb-connection-columns-to-tenant
author: maverick
changes:
- addColumn:
tableName: tenant
columns:
- column:
name: mongodb_host
type: VARCHAR(255)
- column:
name: mongodb_port
type: VARCHAR(255)
- column:
name: mongodb_username
type: VARCHAR(255)
- column:
name: mongodb_password
type: VARCHAR(255)
- column:
name: mongodb_database
type: VARCHAR(255)

View File

@ -20,12 +20,15 @@ import com.knecon.fforesight.tenantusermanagement.feigntestclients.external.Tena
import com.knecon.fforesight.tenantusermanagement.feigntestclients.internal.InternalTenantsClient;
import com.knecon.fforesight.tenantusermanagement.testcontainers.KeyCloakTestContainer;
import com.knecon.fforesight.tenantusermanagement.testcontainers.MinioTestContainer;
import com.knecon.fforesight.tenantusermanagement.testcontainers.MongoDBTestContainer;
import com.knecon.fforesight.tenantusermanagement.testcontainers.RedisTestContainer;
import com.knecon.fforesight.tenantusermanagement.testcontainers.SpringPostgreSQLTestContainer;
import com.knecon.fforesight.tenantusermanagement.utils.TestTenantService;
import com.knecon.fforesight.tenantusermanagement.utils.TokenService;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
@ -70,14 +73,21 @@ public class AbstractTenantUserManagementIntegrationTest {
var kcInstance = KeyCloakTestContainer.getInstance();
var mongoInstance = MongoDBTestContainer.getInstance();
mongoInstance.start();
TestPropertyValues.of("spring.datasource.url=" + postgreSQLContainerMaster.getJdbcUrl() + connectionStringDetails,
"spring.datasource.username=" + postgreSQLContainerMaster.getUsername(),
"spring.datasource.password=" + postgreSQLContainerMaster.getPassword(),
"fforesight.tenant-user-management.serverUrl=" + kcInstance.getAuthServerUrl(),
"REDIS_PORT=" + redisContainer.getFirstMappedPort(),
"REDIS_HOST=" + redisContainer.getHost(),
"fforesight.jobs.enabled=false",
"fforesight.keycloak.enabled=true").applyTo(configurableApplicationContext.getEnvironment());
"spring.datasource.username=" + postgreSQLContainerMaster.getUsername(),
"spring.datasource.password=" + postgreSQLContainerMaster.getPassword(),
"fforesight.tenant-user-management.serverUrl=" + kcInstance.getAuthServerUrl(),
"REDIS_PORT=" + redisContainer.getFirstMappedPort(),
"REDIS_HOST=" + redisContainer.getHost(),
"MONGODB_HOST=" + mongoInstance.getHost(),
"MONGODB_PORT=" + mongoInstance.getFirstMappedPort(),
"MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME,
"MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD,
"fforesight.jobs.enabled=false",
"fforesight.keycloak.enabled=true").applyTo(configurableApplicationContext.getEnvironment());
var minioInstance = MinioTestContainer.getInstance();
minioInstance.start();

View File

@ -0,0 +1,35 @@
package com.knecon.fforesight.tenantusermanagement.testcontainers;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.DockerImageName;
public final class MongoDBTestContainer extends GenericContainer<MongoDBTestContainer> {
private static final String IMAGE_VERSION = "mongo:7.0.2";
public static final Integer MONGO_PORT = 27017;
public static final String MONGO_DATABASE = "mongo_database";
public static final String MONGO_PASSWORD = "mongo_password";
public static final String MONGO_USERNAME = "mongo_username";
private static MongoDBTestContainer mongoDB;
private MongoDBTestContainer() {
super(DockerImageName.parse(IMAGE_VERSION));
}
public static MongoDBTestContainer getInstance() {
if (mongoDB == null) {
mongoDB = new MongoDBTestContainer().withEnv("MONGO_INITDB_ROOT_USERNAME", MONGO_USERNAME)
.withEnv("MONGO_INITDB_ROOT_PASSWORD", MONGO_PASSWORD)
.withEnv("MONGO_INITDB_DATABASE", MONGO_DATABASE)
.withExposedPorts(MONGO_PORT);
}
return mongoDB;
}
}

View File

@ -8,6 +8,7 @@ import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import com.knecon.fforesight.tenantusermanagement.AbstractTenantUserManagementIntegrationTest;
import com.knecon.fforesight.tenantusermanagement.feigntestclients.external.TenantsClient;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -19,8 +20,11 @@ import com.knecon.fforesight.tenantusermanagement.model.SearchConnectionRequest;
import com.knecon.fforesight.tenantusermanagement.model.TenantRequest;
import com.knecon.fforesight.tenantusermanagement.service.RealmService;
import com.knecon.fforesight.tenantusermanagement.utils.TestTenantService;
import feign.FeignException;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.regions.Region;
public class TenantsTest extends AbstractTenantUserManagementIntegrationTest {
@ -47,7 +51,9 @@ public class TenantsTest extends AbstractTenantUserManagementIntegrationTest {
var tenant = tenantsClient.getTenant("new_tenant");
assertThat(tenant.getGuid()).isNotBlank();
assertThat(tenantsClient.getTenants().stream().anyMatch(t -> t.getTenantId().equals("new_tenant"))).isTrue();
assertThat(tenantsClient.getTenants()
.stream()
.anyMatch(t -> t.getTenantId().equals("new_tenant"))).isTrue();
var realm = realmService.realm("master").toRepresentation();
assertThat(realm.getLoginTheme()).isEqualTo("redaction");
@ -68,42 +74,45 @@ public class TenantsTest extends AbstractTenantUserManagementIntegrationTest {
.tenantId(tenantId)
.displayName("updated_display_name")
.searchConnection(SearchConnectionRequest.builder()
.numberOfReplicas("1")
.hosts(Set.of("updated_host"))
.numberOfShards("2")
.password("updated_pwd")
.port(123)
.username("updated_username")
.scheme("updated_scheme")
.build())
.numberOfReplicas("1")
.hosts(Set.of("updated_host"))
.numberOfShards("2")
.password("updated_pwd")
.port(123)
.username("updated_username")
.scheme("updated_scheme")
.build())
.databaseConnection(DatabaseConnection.builder()
.database("updated_db")
.driver("updated_driver")
.host("updated_host")
.port("123")
.schema("updated_schema")
.params(Map.of("key", "value"))
.password("updated_pwd")
.username("updated_username")
.build())
.database("updated_db")
.driver("updated_driver")
.host("updated_host")
.port("123")
.schema("updated_schema")
.params(Map.of("key", "value"))
.password("updated_pwd")
.username("updated_username")
.build())
.s3StorageConnection(S3StorageConnection.builder()
.key("minioadmin")
.secret("minioadmin")
.bucketName("redaction2")
.region(Region.AWS_GLOBAL.id())
.endpoint("http://localhost:" + minioPort)
.build())
.key("minioadmin")
.secret("minioadmin")
.bucketName("redaction2")
.region(Region.AWS_GLOBAL.id())
.endpoint("http://localhost:" + minioPort)
.build())
.mongoDBConnection(MongoDBConnection.builder().host("updated_host").port("123").password("updated_pwd").username("updated_username").database("updated_db").build())
.build();
var updatedTenant = tenantsClient.updateTenant(tenantId, tenantRequest);
tenantRequest.getSearchConnection().setPassword(PASSWORD);
tenantRequest.getDatabaseConnection().setPassword(PASSWORD);
tenantRequest.getMongoDBConnection().setPassword(PASSWORD);
assertThat(updatedTenant.getDisplayName()).isEqualTo(tenantRequest.getDisplayName());
assertThat(updatedTenant.getSearchConnection()).isEqualTo(convert(tenantRequest.getSearchConnection(), tenantId));
assertThat(updatedTenant.getDatabaseConnection()).isEqualTo(tenantRequest.getDatabaseConnection());
assertThat(updatedTenant.getS3StorageConnection().getBucketName()).isEqualTo("redaction2");
assertThat(updatedTenant.getMongoDBConnection()).isEqualTo(tenantRequest.getMongoDBConnection());
var tenantEntity = tenantsClient.getTenant(tenantId);
@ -112,6 +121,7 @@ public class TenantsTest extends AbstractTenantUserManagementIntegrationTest {
assertThat(tenantEntity.getDatabaseConnection()).isEqualTo(tenantRequest.getDatabaseConnection());
assertThat(tenantEntity.getS3StorageConnection().getBucketName()).isEqualTo("redaction2");
assertThat(tenantEntity.getS3StorageConnection().getSecret()).isEqualTo(PASSWORD);
assertThat(tenantEntity.getMongoDBConnection()).isEqualTo(tenantRequest.getMongoDBConnection());
tenantRequest.setS3StorageConnection(null);
@ -147,13 +157,13 @@ public class TenantsTest extends AbstractTenantUserManagementIntegrationTest {
var tenantRequest = TenantRequest.builder()
.tenantId("new_tenant")
.s3StorageConnection(S3StorageConnection.builder()
.key("updated_key")
.bucketName("updated_bucket")
.endpoint("updated_endpoint")
.region("updated_region")
.secret("updated_secret")
.signerType("updated_signer")
.build())
.key("updated_key")
.bucketName("updated_bucket")
.endpoint("updated_endpoint")
.region("updated_region")
.secret("updated_secret")
.signerType("updated_signer")
.build())
.build();
var exception = assertThrows(FeignException.BadRequest.class, () -> tenantsClient.updateTenant("new_tenant", tenantRequest));
@ -191,13 +201,13 @@ public class TenantsTest extends AbstractTenantUserManagementIntegrationTest {
.tenantId("new_tenant")
.azureStorageConnection(AzureStorageConnection.builder().connectionString("updated_connection").containerName("updated_container").build())
.s3StorageConnection(S3StorageConnection.builder()
.key("updated_key")
.bucketName("updated_bucket")
.endpoint("updated_endpoint")
.region("updated_region")
.secret("updated_secret")
.signerType("updated_signer")
.build())
.key("updated_key")
.bucketName("updated_bucket")
.endpoint("updated_endpoint")
.region("updated_region")
.secret("updated_secret")
.signerType("updated_signer")
.build())
.build();
var exception = assertThrows(FeignException.BadRequest.class, () -> tenantsClient.updateTenant("new_tenant", tenantRequest));
@ -239,6 +249,8 @@ public class TenantsTest extends AbstractTenantUserManagementIntegrationTest {
assertThat(tenant.getSearchConnection().getPassword()).isEqualTo(PASSWORD);
assertThat(tenant.getS3StorageConnection()).isNotNull();
assertThat(tenant.getS3StorageConnection().getSecret()).isEqualTo(PASSWORD);
assertThat(tenant.getMongoDBConnection()).isNotNull();
assertThat(tenant.getMongoDBConnection().getPassword()).isEqualTo(PASSWORD);
TenantContext.clear();
}

View File

@ -8,6 +8,7 @@ import java.util.UUID;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import com.knecon.fforesight.tenantusermanagement.feigntestclients.external.TenantsClient;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.model.DatabaseConnection;
@ -16,7 +17,9 @@ import com.knecon.fforesight.tenantusermanagement.api.internal.InternalTenantsRe
import com.knecon.fforesight.tenantusermanagement.model.SearchConnectionRequest;
import com.knecon.fforesight.tenantusermanagement.model.TenantRequest;
import com.knecon.fforesight.tenantusermanagement.model.TenantUser;
import com.knecon.fforesight.tenantusermanagement.testcontainers.MongoDBTestContainer;
import com.knecon.fforesight.tenantusermanagement.testcontainers.SpringPostgreSQLTestContainer;
import lombok.RequiredArgsConstructor;
import software.amazon.awssdk.regions.Region;
@ -53,30 +56,31 @@ public class TestTenantService {
.guid(UUID.randomUUID().toString())
.defaultUsers(List.of(TenantUser.builder().roles(Set.of("SUPER_USER")).username("test@fforesight.com").password("secret").email("test@fforesight.com").build()))
.databaseConnection(DatabaseConnection.builder()
.driver("postgresql")
.host(SpringPostgreSQLTestContainer.getInstance().getHost())
.port(String.valueOf(SpringPostgreSQLTestContainer.getInstance().getFirstMappedPort()))
.database(SpringPostgreSQLTestContainer.getInstance().getDatabaseName())
.schema(testTenantId)
.username(SpringPostgreSQLTestContainer.getInstance().getUsername())
.password(SpringPostgreSQLTestContainer.getInstance().getPassword())
.build())
.searchConnection(SearchConnectionRequest.builder()
.hosts(Set.of("localhost"))
.port(9200)
.scheme("http")
.numberOfShards("1")
.numberOfReplicas("5")
.build());
.driver("postgresql")
.host(SpringPostgreSQLTestContainer.getInstance().getHost())
.port(String.valueOf(SpringPostgreSQLTestContainer.getInstance().getFirstMappedPort()))
.database(SpringPostgreSQLTestContainer.getInstance().getDatabaseName())
.schema(testTenantId)
.username(SpringPostgreSQLTestContainer.getInstance().getUsername())
.password(SpringPostgreSQLTestContainer.getInstance().getPassword())
.build())
.searchConnection(SearchConnectionRequest.builder().hosts(Set.of("localhost")).port(9200).scheme("http").numberOfShards("1").numberOfReplicas("5").build())
.mongoDBConnection(MongoDBConnection.builder()
.host(MongoDBTestContainer.getInstance().getHost())
.port(String.valueOf(MongoDBTestContainer.getInstance().getFirstMappedPort()))
.database(MongoDBTestContainer.MONGO_DATABASE)
.username(MongoDBTestContainer.MONGO_USERNAME)
.password(MongoDBTestContainer.MONGO_PASSWORD)
.build());
if (withStorage) {
tenantRequest = tenantRequestBuilder.s3StorageConnection(S3StorageConnection.builder()
.key("minioadmin")
.secret("minioadmin")
.bucketName(testTenantId.replaceAll("-", "").replaceAll("_", ""))
.endpoint("http://localhost:" + actualPort)
.region(Region.AWS_GLOBAL.id())
.build()).build();
.key("minioadmin")
.secret("minioadmin")
.bucketName(testTenantId.replaceAll("-", "").replaceAll("_", ""))
.endpoint("http://localhost:" + actualPort)
.region(Region.AWS_GLOBAL.id())
.build()).build();
} else {
tenantRequest = tenantRequestBuilder.build();
}