diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java index 53e4206c8..580b42302 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/PersistenceServiceProcessorConfiguration.java @@ -1,11 +1,17 @@ package com.iqser.red.service.persistence.management.v1.processor; +import javax.sql.DataSource; + +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.cloud.openfeign.support.PageJacksonModule; import org.springframework.cloud.openfeign.support.SortJacksonModule; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.retry.backoff.ExponentialBackOffPolicy; import org.springframework.retry.policy.SimpleRetryPolicy; import org.springframework.retry.support.RetryTemplate; @@ -21,6 +27,9 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa @EnableFeignClients(basePackageClasses = {PDFTronClient.class, StatusReportClient.class, SearchClient.class, RedactionClient.class}) public class PersistenceServiceProcessorConfiguration { + public static final String TENANT_DATA_SOURCE_QUALIFIER = "multiTenantDataSource"; + + @Bean public PageJacksonModule pageJacksonModule() { @@ -51,4 +60,13 @@ public class PersistenceServiceProcessorConfiguration { return retryTemplate; } + + @Bean + @DependsOn(TENANT_DATA_SOURCE_QUALIFIER) + @Primary + public JdbcTemplate tenantJdbcTemplate(@Qualifier(TENANT_DATA_SOURCE_QUALIFIER) DataSource dataSource) { + + return new JdbcTemplate(dataSource); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCUtils.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCUtils.java index 46c3b89fc..5aecbbbbd 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCUtils.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCUtils.java @@ -1,5 +1,10 @@ package com.iqser.red.service.persistence.management.v1.processor.utils.jdbc; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + import com.iqser.red.service.persistence.management.v1.processor.multitenancy.entity.DatabaseConnectionEntity; import com.iqser.red.service.persistence.service.v1.api.shared.model.multitenancy.DatabaseConnection; @@ -8,75 +13,87 @@ import lombok.experimental.UtilityClass; @UtilityClass public class JDBCUtils { + public String buildJdbcUrl(DatabaseConnectionEntity databaseConnectionEntity) { - public String buildJdbcUrl(DatabaseConnectionEntity databaseConnectionEntity){ - StringBuilder sb = new StringBuilder("jdbc:") - .append(databaseConnectionEntity.getDriver()) - .append("://") - .append(databaseConnectionEntity.getHost()) - .append(':') - .append(databaseConnectionEntity.getPort()) - .append('/') - .append(databaseConnectionEntity.getDatabase()); + StringBuilder sb = createJdbcConnectionStringBuilder(databaseConnectionEntity.getDriver(), + databaseConnectionEntity.getHost(), + databaseConnectionEntity.getPort(), + databaseConnectionEntity.getDatabase()); - if(databaseConnectionEntity.getParams() != null) { - sb.append('?'); - databaseConnectionEntity.getParams().forEach((k, v) -> sb.append('&').append(k).append(v)); - } + Map params = getConnectionParameters(databaseConnectionEntity); + appendParams(sb, params); return sb.toString(); } - public String buildJdbcUrl(DatabaseConnection databaseConnection){ - StringBuilder sb = new StringBuilder("jdbc:") - .append(databaseConnection.getDriver()) - .append("://") - .append(databaseConnection.getHost()) - .append(':') - .append(databaseConnection.getPort()) - .append('/') - .append(databaseConnection.getDatabase()); - if(databaseConnection.getParams() != null) { - sb.append('?'); - databaseConnection.getParams().forEach((k, v) -> sb.append('&').append(k).append(v)); - } + private StringBuilder createJdbcConnectionStringBuilder(String driver, String host, String port, String database) { + + return new StringBuilder("jdbc:").append(driver).append("://").append(host).append(':').append(port).append('/').append(database); + } + + + private Map getConnectionParameters(DatabaseConnectionEntity databaseConnectionEntity) { + + return Optional.ofNullable(databaseConnectionEntity.getParams()).orElseGet(HashMap::new); + } + + + public String buildJdbcUrl(DatabaseConnection databaseConnection) { + + StringBuilder sb = createJdbcConnectionStringBuilder(databaseConnection.getDriver(), + databaseConnection.getHost(), + databaseConnection.getPort(), + databaseConnection.getDatabase()); + Map params = getConnectionParameters(databaseConnection); + appendParams(sb, params); return sb.toString(); } - public String buildJdbcUrlWithSchema(DatabaseConnection databaseConnection){ - StringBuilder sb = new StringBuilder("jdbc:") - .append(databaseConnection.getDriver()) - .append("://") - .append(databaseConnection.getHost()) - .append(':') - .append(databaseConnection.getPort()) - .append('/') - .append(databaseConnection.getDatabase()) - .append('?') - .append("currentSchema=") - .append(databaseConnection.getSchema()); - if(databaseConnection.getParams() != null) { - databaseConnection.getParams().forEach((k, v) -> sb.append('&').append(k).append(v)); + private void appendParams(StringBuilder sb, Map params) { + + if (!params.isEmpty()) { + sb.append("?"); + List paramsAsStrings = params.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).toList(); + sb.append(String.join("&", paramsAsStrings)); } + } + + + public String buildJdbcUrlWithSchema(DatabaseConnection databaseConnection) { + + return createJdbcConnectionString(databaseConnection.getDriver(), + databaseConnection.getHost(), + databaseConnection.getPort(), + databaseConnection.getDatabase(), + databaseConnection.getSchema(), + getConnectionParameters(databaseConnection)); + } + + + private static String createJdbcConnectionString(String driver, String host, String port, String database, String schema, Map connectionParameters) { + + StringBuilder sb = createJdbcConnectionStringBuilder(driver, host, port, database); + connectionParameters.put("currentSchema", schema); + appendParams(sb, connectionParameters); return sb.toString(); } - public String buildJdbcUrlWithSchema(DatabaseConnectionEntity databaseConnection){ - StringBuilder sb = new StringBuilder("jdbc:") - .append(databaseConnection.getDriver()) - .append("://") - .append(databaseConnection.getHost()) - .append(':') - .append(databaseConnection.getPort()) - .append('/') - .append(databaseConnection.getDatabase()) - .append('?') - .append("currentSchema=") - .append(databaseConnection.getSchema()); - if(databaseConnection.getParams() != null) { - databaseConnection.getParams().forEach((k, v) -> sb.append('&').append(k).append(v)); - } - return sb.toString(); + + private Map getConnectionParameters(DatabaseConnection databaseConnection) { + + return Optional.ofNullable(databaseConnection.getParams()).orElseGet(HashMap::new); } + + + public String buildJdbcUrlWithSchema(DatabaseConnectionEntity databaseConnection) { + + return createJdbcConnectionString(databaseConnection.getDriver(), + databaseConnection.getHost(), + databaseConnection.getPort(), + databaseConnection.getDatabase(), + databaseConnection.getSchema(), + getConnectionParameters(databaseConnection)); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java index e75bf3a22..7c71a2a32 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java @@ -12,20 +12,14 @@ import java.util.Map; import java.util.stream.Collectors; import javax.persistence.Column; -import javax.persistence.EntityManager; import javax.persistence.Table; import javax.transaction.Transactional; import org.apache.commons.lang3.StringUtils; -import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.core.env.Environment; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceUtils; -import org.springframework.jdbc.datasource.SingleConnectionDataSource; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.mulitenancy.DynamicDataSourceBasedMultiTenantConnectionProvider; -import com.iqser.red.service.persistence.management.v1.processor.utils.multitenancy.TenantContext; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -35,75 +29,64 @@ import lombok.SneakyThrows; @RequiredArgsConstructor public class JDBCWriteUtils { - private final String SQL_TEMPLATE = "INSERT INTO %s (%s) values (%s)"; + private static final String SQL_TEMPLATE = "INSERT INTO %s (%s) values (%s)"; private final Map, EntityMetadata> entityMetadataMap = new HashMap<>(); - private final EntityManager entityManager; + private final JdbcTemplate jdbcTemplate; + + private final Environment environment; @SneakyThrows @Transactional - public void saveBatch(final List entities) { + public void saveBatch(List entities) { if (entities.isEmpty()) { return; } - var metadata = getEntityMetadata(entities.iterator().next()); + var metadata = getEntityMetadata(entities.get(0).getClass()); - final int batchSize = 500; - - var query = entityManager.createNativeQuery(metadata.getSqlStatement()); - for (int j = 0; j < entities.size(); j += batchSize) { - - final List batchList = entities.subList(j, Math.min(j + batchSize, entities.size())); - - for (var entity : batchList) { - int paramIndex = 1; - for (var mapping : metadata.getFieldMethodMap().entrySet()) { - query.setParameter(paramIndex++, mapping.getValue().invoke(entity)); - } - - query.executeUpdate(); - - } - - } - - entityManager.clear(); + jdbcTemplate.batchUpdate(metadata.getSqlStatement(), entities, getBatchSize(), metadata::setValues); } - private EntityMetadata getEntityMetadata(T entity) { + private int getBatchSize() { - var existingMetadata = entityMetadataMap.get(entity.getClass()); + return environment.getProperty("spring.jpa.properties.hibernate.jdbc.batch_size", int.class, 500); + } + + + private EntityMetadata getEntityMetadata(Class entityClass) { + + var existingMetadata = entityMetadataMap.get(entityClass); if (existingMetadata != null) { return existingMetadata; } - var tableName = getTableName(entity); - var args = getArgs(entity); + var tableName = getTableName(entityClass); + var args = getArgs(entityClass); var sql = String.format(SQL_TEMPLATE, tableName, String.join(", ", args.keySet()), args.keySet().stream().map(a -> "?").collect(Collectors.joining(", "))); var metadata = new EntityMetadata(tableName, sql, args); - entityMetadataMap.put(entity.getClass(), metadata); + entityMetadataMap.put(entityClass, metadata); return metadata; } - private String getTableName(T entity) { + private String getTableName(Class entityClass) { - var tableAnnot = entity.getClass().getDeclaredAnnotation(Table.class); + var tableAnnot = entityClass.getDeclaredAnnotation(Table.class); return tableAnnot.name(); } @SneakyThrows - private Map getArgs(T entity) { + private Map getArgs(Class entityClass) { - var fields = entity.getClass().getDeclaredFields(); + var fields = entityClass.getDeclaredFields(); Map entityMethodMap = new LinkedHashMap<>(); for (var field : fields) { var annotations = field.getDeclaredAnnotations(); @@ -111,7 +94,7 @@ public class JDBCWriteUtils { if (annotation.annotationType().equals(Column.class)) { var columnAnnotation = (Column) annotation; var name = StringUtils.isEmpty(columnAnnotation.name()) ? toSnakeCase(field.getName()) : columnAnnotation.name(); - entityMethodMap.put(name, entity.getClass().getMethod(getMethodName(field))); + entityMethodMap.put(name, entityClass.getMethod(getMethodName(field))); } } } @@ -144,6 +127,16 @@ public class JDBCWriteUtils { private String sqlStatement; private Map fieldMethodMap; + + @SneakyThrows + public void setValues(PreparedStatement preparedStatement, Object entity) { + + int paramIndex = 1; + for (var mapping : getFieldMethodMap().entrySet()) { + preparedStatement.setObject(paramIndex++, mapping.getValue().invoke(entity)); + } + } + } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml index b8771cbb0..fc2e39edd 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml @@ -22,7 +22,7 @@ spring: properties: hibernate: jdbc: - batch_size: 50 + batch_size: 1000 order_inserts: true order_updates: true cache: diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java index c5d5931a6..c80ec4019 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java @@ -101,33 +101,33 @@ public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest @Test public void testWritePerformance() { - var tenKEntries = generateEntries(10_000); + final int numberOfEntries = 10_000; + var entries = generateEntries(numberOfEntries); var template = dossierTemplateTesterAndProvider.provideTestTemplate("test"); var type1 = typeProvider.testAndProvideType(template, null, "t1"); var type2 = typeProvider.testAndProvideType(template, null, "t2"); - List type1Entries = tenKEntries.stream().map(s -> new DictionaryEntryEntity(0, s, 1, false, type1.getTypeId())).collect(Collectors.toList()); + List type1Entries = entries.stream().map(s -> new DictionaryEntryEntity(0, s, 1, false, type1.getTypeId())).collect(Collectors.toList()); - List type2Entries = tenKEntries.stream().map(s -> new DictionaryEntryEntity(0, s, 1, false, type2.getTypeId())).collect(Collectors.toList()); + List type2Entries = entries.stream().map(s -> new DictionaryEntryEntity(0, s, 1, false, type2.getTypeId())).collect(Collectors.toList()); assertThat(entryRepository.count()).isEqualTo(0); long t1 = System.currentTimeMillis(); entryRepository.saveAll(type1Entries); long jpaTime = System.currentTimeMillis() - t1; - assertThat(entryRepository.count()).isEqualTo(10_000); + assertThat(entryRepository.count()).isEqualTo(numberOfEntries); t1 = System.currentTimeMillis(); jdbcWriteUtils.saveBatch(type2Entries); long jdbcTime = System.currentTimeMillis() - t1; - assertThat(entryRepository.count()).isEqualTo(20_000); + assertThat(entryRepository.count()).isEqualTo(2 * numberOfEntries); // assertThat(jpaTime).isGreaterThan(jdbcTime); - System.out.println("JPA Time: " + jpaTime + "ms for 10k entries"); - System.out.println("JDBC Time: " + jdbcTime + "ms for 10k entries"); - + log.info("JPA Time: {} ms for {} entries", jpaTime, numberOfEntries); + log.info("JDBC Time: {} ms for {} entries", jdbcTime, numberOfEntries); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml index 87063e89f..d61dfe710 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml @@ -16,7 +16,6 @@ spring: scheduler: instanceId: AUTO job-store-type: JDBC - main: allow-circular-references: true # FIXME jpa: @@ -28,7 +27,7 @@ spring: properties: hibernate: jdbc: - batch_size: 50 + batch_size: 1000 order_inserts: true order_updates: true open-in-view: true