Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'master' into RED-6501

Browse Source
This commit is contained in:
Viktor Seifert 2023-04-05 12:07:09 +02:00
commit 45d23e3ff4
13 changed files with 314 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/multitenancy/persistence/TenantPersistenceConfig.java
View File

@ -28,12 +28,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.mulitenancy.DynamicDataSourceBasedMultiTenantConnectionProvider;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.mulitenancy.repository.TenantRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ColorsRepository;
import com.iqser.red.service.persistence.management.v1.processor.settings.TenantHikariSettings;
import lombok.RequiredArgsConstructor;
@Configuration
@EnableJpaRepositories(basePackageClasses = ColorsRepository.class, entityManagerFactoryRef = "tenantEntityManagerFactory", transactionManagerRef = "tenantTransactionManager")
@EnableConfigurationProperties(JpaProperties.class)
@EnableConfigurationProperties({JpaProperties.class, TenantHikariSettings.class})
@RequiredArgsConstructor
public class TenantPersistenceConfig {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/TenantManagementService.java
View File

@ -87,7 +87,6 @@ public class TenantManagementService {
private final KeyCloakRoleManagerService keyCloakRoleManagerService;
private final KeyCloakAdminClientService keycloak;
public TenantManagementService(EncryptionDecryptionService encryptionService,
@Qualifier("tenantLiquibaseProperties") LiquibaseProperties liquibaseProperties,
ResourceLoader resourceLoader,
@ -299,6 +298,9 @@ public class TenantManagementService {
var user = new UserRepresentation();
user.setUsername(redUser.getUsername());
user.setCredentials(List.of(credentialRepresentation));
user.setEmail(redUser.getEmail());
user.setFirstName(redUser.getFirstName());
user.setLastName(redUser.getLastName());
user.setEmailVerified(true);
var roles = new ArrayList<String>();

82
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/mulitenancy/DynamicDataSourceBasedMultiTenantConnectionProvider.java
View File

@ -19,8 +19,9 @@ import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
import com.iqser.red.service.persistence.management.v1.processor.multitenancy.entity.TenantEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.EncryptionDecryptionService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.mulitenancy.repository.SchemaDataSource;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.mulitenancy.repository.SchemaConnection;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.mulitenancy.repository.TenantRepository;
import com.iqser.red.service.persistence.management.v1.processor.settings.TenantHikariSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.jdbc.JDBCUtils;
import com.zaxxer.hikari.HikariDataSource;
@ -38,6 +39,7 @@ public class DynamicDataSourceBasedMultiTenantConnectionProvider extends Abstrac
private final DataSourceProperties masterDataSourceProperties;
private final TenantRepository masterTenantRepository;
private final EncryptionDecryptionService encryptionService;
private final TenantHikariSettings tenantHikariSettings;
@Value("${multitenancy.datasource-cache.maximumSize:100}")
private Long maximumSize;
@ -45,64 +47,57 @@ public class DynamicDataSourceBasedMultiTenantConnectionProvider extends Abstrac
@Value("${multitenancy.datasource-cache.expireAfterAccess:10}")
private Integer expireAfterAccess;
private LoadingCache<String, SchemaDataSource> tenantSchemaDataSources;
private LoadingCache<String, SchemaConnection> connectionPerTenant;
private LoadingCache<SchemaConnection, DataSource> dataSourcePerConnectionString;
@PostConstruct
protected void createCache() {
tenantSchemaDataSources = CacheBuilder.newBuilder()
connectionPerTenant = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(expireAfterAccess, TimeUnit.MINUTES).build(new CacheLoader<>() {
public SchemaConnection load(String key) {
TenantEntity tenant = masterTenantRepository.findByTenantId(key).orElseThrow(() -> new RuntimeException("No such tenant: " + key));
var jdbcUrl = JDBCUtils.buildJdbcUrl(tenant.getDatabaseConnection());
return SchemaConnection.builder().jdbcUrl(jdbcUrl).databaseConnection(tenant.getDatabaseConnection()).build();
}
});
dataSourcePerConnectionString = CacheBuilder.newBuilder()
.maximumSize(maximumSize)
.expireAfterAccess(expireAfterAccess, TimeUnit.MINUTES)
.removalListener((RemovalListener<String, SchemaDataSource>) removal -> {
var toRemove = removal.getValue();
int numberOfSchemasForSameDataSource = 0;
for (var schemaDataSource : tenantSchemaDataSources.asMap().values()) {
if (toRemove.getJdbcUrl().equals(schemaDataSource.getJdbcUrl())) {
numberOfSchemasForSameDataSource++;
.removalListener((RemovalListener<SchemaConnection, DataSource>) removal -> {
HikariDataSource ds = (HikariDataSource) removal.getValue();
ds.close();
log.info("Closed datasource: {}", ds.getPoolName());
}
}
if (numberOfSchemasForSameDataSource == 0) {
HikariDataSource ds = (HikariDataSource) removal.getValue().getDataSource();
ds.close(); // tear down properly
log.info("Closed datasource: {}", ds.getPoolName());
} else {
log.info("Keeping datasource open from {} because it is still used by {} other tenants", removal.getKey(), numberOfSchemasForSameDataSource);
}
})
)
.build(new CacheLoader<>() {
public SchemaDataSource load(String key) {
public DataSource load(SchemaConnection schemaConnection) {
TenantEntity tenant = masterTenantRepository.findByTenantId(key).orElseThrow(() -> new RuntimeException("No such tenant: " + key));
var jdbcUrl = JDBCUtils.buildJdbcUrl(tenant.getDatabaseConnection());
for (var schemaDataSource : tenantSchemaDataSources.asMap().values()) {
if (schemaDataSource.getJdbcUrl().equals(jdbcUrl)) {
return new SchemaDataSource(tenant.getDatabaseConnection().getSchema(), schemaDataSource.getDataSource(), jdbcUrl);
}
}
return createAndConfigureDataSource(tenant);
return createAndConfigureDataSource(schemaConnection);
}
});
}
private SchemaDataSource createAndConfigureDataSource(TenantEntity tenant) {
private DataSource createAndConfigureDataSource(SchemaConnection connection) {
String decryptedPassword = encryptionService.decrypt(tenant.getDatabaseConnection().getPassword());
String decryptedPassword = encryptionService.decrypt(connection.getDatabaseConnection().getPassword());
HikariDataSource ds = masterDataSourceProperties.initializeDataSourceBuilder().type(HikariDataSource.class).build();
tenantHikariSettings.setUsername(connection.getDatabaseConnection().getUsername());
tenantHikariSettings.setPassword(decryptedPassword);
ds.setUsername(tenant.getDatabaseConnection().getUsername());
ds.setPassword(decryptedPassword);
var jdbcUrl = JDBCUtils.buildJdbcUrl(connection.getDatabaseConnection());
tenantHikariSettings.setJdbcUrl(jdbcUrl);
tenantHikariSettings.setPoolName(jdbcUrl + TENANT_POOL_NAME_SUFFIX);
var jdbcUrl = JDBCUtils.buildJdbcUrl(tenant.getDatabaseConnection());
ds.setJdbcUrl(jdbcUrl);
ds.setPoolName(jdbcUrl + TENANT_POOL_NAME_SUFFIX);
HikariDataSource ds = new HikariDataSource(tenantHikariSettings);
log.info("Configured datasource: {}", ds.getPoolName());
return new SchemaDataSource(tenant.getDatabaseConnection().getSchema(), ds, jdbcUrl);
return ds;
}
@ -117,7 +112,8 @@ public class DynamicDataSourceBasedMultiTenantConnectionProvider extends Abstrac
public DataSource selectDataSource(String tenantIdentifier) {
try {
return tenantSchemaDataSources.get(tenantIdentifier).getDataSource();
var connection = connectionPerTenant.get(tenantIdentifier);
return dataSourcePerConnectionString.get(connection);
} catch (ExecutionException e) {
throw new RuntimeException("Failed to load DataSource for tenant: " + tenantIdentifier);
}
@ -128,9 +124,9 @@ public class DynamicDataSourceBasedMultiTenantConnectionProvider extends Abstrac
public Connection getConnection(String tenantIdentifier) throws SQLException {
try {
var schemaDataSource = tenantSchemaDataSources.get(tenantIdentifier);
Connection connection = schemaDataSource.getDataSource().getConnection();
connection.setSchema(schemaDataSource.getSchema());
var dataSource = selectDataSource(tenantIdentifier);
Connection connection = dataSource.getConnection();
connection.setSchema(connectionPerTenant.get(tenantIdentifier).getDatabaseConnection().getSchema());
return connection;
} catch (ExecutionException e) {
throw new RuntimeException("No such tenant: " + tenantIdentifier);
@ -140,6 +136,7 @@ public class DynamicDataSourceBasedMultiTenantConnectionProvider extends Abstrac
@Override
public void releaseAnyConnection(Connection connection) throws SQLException {
connection.close();
}
@ -154,6 +151,7 @@ public class DynamicDataSourceBasedMultiTenantConnectionProvider extends Abstrac
@Override
public boolean supportsAggressiveRelease() {
return false;
}

20
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/mulitenancy/repository/SchemaConnection.java Normal file
View File

@ -0,0 +1,20 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.mulitenancy.repository;
import com.iqser.red.service.persistence.management.v1.processor.multitenancy.entity.DatabaseConnectionEntity;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
@Data
@Builder
@AllArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class SchemaConnection {
@EqualsAndHashCode.Include
private String jdbcUrl;
private DatabaseConnectionEntity databaseConnection;
}

18
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/mulitenancy/repository/SchemaDataSource.java
View File

@ -1,18 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.mulitenancy.repository;
import javax.sql.DataSource;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
@AllArgsConstructor
public class SchemaDataSource {
private String schema;
private DataSource dataSource;
private String jdbcUrl;
}

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/settings/TenantHikariSettings.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.settings;
import org.springframework.boot.context.properties.ConfigurationProperties;
import com.zaxxer.hikari.HikariConfig;
@ConfigurationProperties("multitenancy.tenant.datasource.hikari")
public class TenantHikariSettings extends HikariConfig {
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-master.yaml
View File

@ -5,3 +5,5 @@ databaseChangeLog:
file: db/changelog/master/2-quartz.changelog.yaml
- include:
file: db/changelog/master/3-detailed-db-connection.changelog.yaml
- include:
file: db/changelog/master/4-add-unique-constraint-for-tenants-table.yaml

17
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/master/4-add-unique-constraint-for-tenants-table.yaml Normal file
View File

@ -0,0 +1,17 @@
databaseChangeLog:
- changeSet:
id: add-unique-constraint-for-tenants-table
author: corinaolariu
changes:
- addUniqueConstraint:
columnNames: db_host, db_schema
constraintName: unique_constraint_tenant_host_shema
tableName: tenant
- addUniqueConstraint:
columnNames: storage_s3_endpoint, storage_s3_region, storage_s3_bucket_name
constraintName: unique_constraint_tenant_s3_storage
tableName: tenant
- addUniqueConstraint:
columnNames: storage_azure_connection_string, storage_azure_container_name
constraintName: unique_constraint_tenant_azure_storage
tableName: tenant

4
persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml
View File

@ -97,6 +97,8 @@ multitenancy:
password: ${PSQL_PASSWORD:redaction}
platform: org.hibernate.dialect.PostgreSQL95Dialect
hikari:
maximumPoolSize: 5
minimum-idle: 5
data-source-properties:
cachePrepStmts: true
prepStmtCacheSize: 1000
@ -107,6 +109,8 @@ multitenancy:
datasource:
driverClassName: org.postgresql.Driver
hikari:
maximumPoolSize: 5
minimum-idle: 5
data-source-properties:
cachePrepStmts: true
prepStmtCacheSize: 1000

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java
View File

@ -8,6 +8,7 @@ import java.util.List;
import java.util.Set;
import java.util.stream.IntStream;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -62,6 +63,7 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
private TypeProvider typeProvider;
@Test
@Disabled // TODO parallel does not work with tenantContext currently
public void testDossierRaceCondition() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

207
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TenantsTest.java Normal file
View File

@ -0,0 +1,207 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.keycloak.commons.roles.ApplicationRoles;
import com.iqser.red.service.peristence.v1.server.integration.client.TenantsClient;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.multitenancy.AzureStorageConnection;
import com.iqser.red.service.persistence.service.v1.api.shared.model.multitenancy.DatabaseConnection;
import com.iqser.red.service.persistence.service.v1.api.shared.model.multitenancy.RedUser;
import com.iqser.red.service.persistence.service.v1.api.shared.model.multitenancy.S3StorageConnection;
import com.iqser.red.service.persistence.service.v1.api.shared.model.multitenancy.SearchConnection;
import com.iqser.red.service.persistence.service.v1.api.shared.model.multitenancy.TenantRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.multitenancy.TenantResponse;
import feign.FeignException;
public class TenantsTest extends AbstractPersistenceServerServiceTest {
@Autowired
private TenantsClient tenantsClient;
@Test
public void testCreateTenantWithSameHostAndSchema() {
TenantResponse firstTenant = tenantsClient.getTenants().get(0);
var tenantRequest = TenantRequest.builder()
.tenantId("redaction2")
.displayName("Redaction default2")
.guid(UUID.randomUUID().toString())
.databaseConnection(DatabaseConnection.builder()
.driver("postgresql")
.host("localhost")
.port(firstTenant.getDatabaseConnection().getPort())
.database("redaction")
.schema("myschema")
.username("redaction")
.password("redaction")
.build())
.searchConnection(SearchConnection.builder()
.hosts(Set.of("elasticsearchHost2"))
.port(9200)
.scheme("https2")
.username("elastic")
.numberOfShards("1")
.numberOfReplicas("5")
.build())
.s3StorageConnection(S3StorageConnection.builder()
.key("key")
.secret("secret")
.signerType("signerType")
.bucketName("bucketName2")
.region("eu")
.endpoint("endpoint2")
.build())
.redUsers(List.of(RedUser.builder().username("user").password("password").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build(),
RedUser.builder().username("manageradmin1@test.com").password("secret").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build(),
RedUser.builder().username("manageradmin2@test.com").password("secret").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build()))
.build();
Exception exception = Assertions.assertThrows(FeignException.Conflict.class, () -> {
tenantsClient.createTenant(tenantRequest);
});
String expectedMessage = "An object of type tenant already exists";
String actualMessage = exception.getMessage();
assertThat(actualMessage).contains(expectedMessage);
}
@Test
public void testCreateTenantWithDuplicateStorageS3() {
TenantResponse firstTenant = tenantsClient.getTenants().get(0);
var tenantRequest = TenantRequest.builder()
.tenantId("redaction2")
.displayName("Redaction default2")
.guid(UUID.randomUUID().toString())
.databaseConnection(DatabaseConnection.builder()
.driver("postgresql")
.host("localhost")
.port(firstTenant.getDatabaseConnection().getPort())
.database("redaction")
.schema("myschema2")
.username("redaction")
.password("redaction")
.build())
.searchConnection(SearchConnection.builder()
.hosts(Set.of("elasticsearchHost2"))
.port(9200)
.scheme("https2")
.username("elastic")
.numberOfShards("1")
.numberOfReplicas("5")
.build())
.s3StorageConnection(S3StorageConnection.builder()
.key("key")
.secret("secret")
.signerType("signerType")
.bucketName("bucketName")
.region("eu")
.endpoint("endpoint")
.build())
.redUsers(List.of(RedUser.builder().username("user").password("password").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build(),
RedUser.builder().username("manageradmin1@test.com").password("secret").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build(),
RedUser.builder().username("manageradmin2@test.com").password("secret").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build()))
.build();
Exception exception = Assertions.assertThrows(FeignException.Conflict.class, () -> {
tenantsClient.createTenant(tenantRequest);
});
String expectedMessage = "An object of type tenant already exists";
String actualMessage = exception.getMessage();
assertThat(actualMessage).contains(expectedMessage);
}
@Test
public void testCreateTenantWithDuplicateAzure() {
TenantResponse firstTenant = tenantsClient.getTenants().get(0);
var tenantRequest = TenantRequest.builder()
.tenantId("redaction2")
.displayName("Redaction default2")
.guid(UUID.randomUUID().toString())
.databaseConnection(DatabaseConnection.builder()
.driver("postgresql")
.host("localhost")
.port(firstTenant.getDatabaseConnection().getPort())
.database("redaction")
.schema("myschema2")
.username("redaction")
.password("redaction")
.build())
.searchConnection(SearchConnection.builder()
.hosts(Set.of("elasticsearchHost2"))
.port(9200)
.scheme("https2")
.username("elastic")
.numberOfShards("1")
.numberOfReplicas("5")
.build())
.azureStorageConnection(AzureStorageConnection.builder()
.connectionString("connection")
.containerName("container")
.build())
.redUsers(List.of(RedUser.builder().username("user").password("password").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build(),
RedUser.builder().username("manageradmin1@test.com").password("secret").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build(),
RedUser.builder().username("manageradmin2@test.com").password("secret").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build()))
.build();
tenantsClient.createTenant(tenantRequest);
assertThat(tenantsClient.getTenants().size()).isEqualTo(2);
var tenantRequest2 = TenantRequest.builder()
.tenantId("redaction2")
.displayName("Redaction default2")
.guid(UUID.randomUUID().toString())
.databaseConnection(DatabaseConnection.builder()
.driver("postgresql")
.host("localhost")
.port(firstTenant.getDatabaseConnection().getPort())
.database("redaction")
.schema("myschema3")
.username("redaction")
.password("redaction")
.build())
.searchConnection(SearchConnection.builder()
.hosts(Set.of("elasticsearchHost2"))
.port(9200)
.scheme("https2")
.username("elastic")
.numberOfShards("1")
.numberOfReplicas("5")
.build())
.azureStorageConnection(AzureStorageConnection.builder()
.connectionString("connection")
.containerName("container")
.build())
.redUsers(List.of(RedUser.builder().username("user").password("password").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build(),
RedUser.builder().username("manageradmin1@test.com").password("secret").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build(),
RedUser.builder().username("manageradmin2@test.com").password("secret").redRoles(ApplicationRoles.ROLE_DATA.keySet()).build()))
.build();
Exception exception = Assertions.assertThrows(FeignException.Conflict.class, () -> {
tenantsClient.createTenant(tenantRequest);
});
String expectedMessage = "An object of type tenant already exists";
String actualMessage = exception.getMessage();
assertThat(actualMessage).contains(expectedMessage);
}
}

4
persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml
View File

@ -96,6 +96,8 @@ multitenancy:
driverClassName: org.postgresql.Driver
platform: org.hibernate.dialect.PostgreSQL95Dialect
hikari:
maximumPoolSize: 5
minimum-idle: 5
data-source-properties:
cachePrepStmts: true
prepStmtCacheSize: 1000
@ -107,6 +109,8 @@ multitenancy:
driverClassName: org.postgresql.Driver
platform: org.hibernate.dialect.PostgreSQL95Dialect
hikari:
maximumPoolSize: 5
minimum-idle: 5
data-source-properties:
cachePrepStmts: true
prepStmtCacheSize: 1000

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/multitenancy/RedUser.java
View File

@ -12,6 +12,9 @@ public class RedUser {
private String username;
private String password;
private String email;
private String firstName;
private String lastName;
@Builder.Default
private Set<String> redRoles = new HashSet<>();