From 83c02409e907f28bc95c9e957f5ea553d6b6724a Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 8 Nov 2023 15:04:04 +0200 Subject: [PATCH 1/5] Integration of sync service --- .dev/docker-compose.yaml | 0 .../persistence-service-processor-v1/pom.xml | 4 ++-- .../src/main/resources/application-dev.yaml | 16 ++++++++++++++++ .../src/main/resources/application.yaml | 4 +++- persistence-service-v1/pom.xml | 2 +- 5 files changed, 22 insertions(+), 4 deletions(-) mode change 100644 => 100755 .dev/docker-compose.yaml diff --git a/.dev/docker-compose.yaml b/.dev/docker-compose.yaml old mode 100644 new mode 100755 diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index 8ff5c0fa3..ef295ab42 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -18,8 +18,8 @@ 0.4.0 0.5.0 0.18.0 - 0.8.0 - 0.13.0 + 0.9.0 + 0.15.0 diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml index c49a6fac7..82da3b633 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml @@ -35,3 +35,19 @@ persistence-service: fforesight: springdoc: auth-server-url: 'http://localhost:8080/auth' + + + jobs: + enabled: true + datasource: + url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:25432}/${PSQL_DATABASE:tenantmanager}?ApplicationName=${spring.application.name:}-scheduler&cachePrepStmts=true&useServerPrepStmts=true&rewriteBatchedStatements=true + driverClassName: org.postgresql.Driver + username: ${PSQL_USERNAME:tenantmanager} + password: ${PSQL_PASSWORD:r3dact3d} + platform: org.hibernate.dialect.PostgreSQL95Dialect + hikari: + maximumPoolSize: 2 + data-source-properties: + cachePrepStmts: true + prepStmtCacheSize: 1000 + prepStmtCacheSqlLimit: 2048 diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml index 42d540984..22c0b5cab 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml @@ -10,7 +10,7 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/ application: type: "RedactManager" rss.component-log.enabled: false - + server: port: 8080 @@ -89,6 +89,7 @@ multitenancy: prepStmtCacheSqlLimit: 2048 liquibase: changeLog: classpath:db/changelog/db.changelog-tenant.yaml + clear-checksums: true bucket4j: @@ -123,6 +124,7 @@ fforesight: user-exchange: name: 'users-exchange' jobs: + enabled: true tenants: remote: true diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index 0bd891dcb..3256096ac 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -6,7 +6,7 @@ com.iqser.red platform-dependency - 2.2.0 + 2.19.0 From 288bbf9f172be1a72463642e5dd7814a2dfaa29a Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 8 Nov 2023 15:05:36 +0200 Subject: [PATCH 2/5] Integrate sync service --- .../processor/migration/SaasMigrationService.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 0b4335c44..cb023ab9a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -16,6 +16,8 @@ import com.iqser.red.service.redaction.v1.model.MigrationRequest; import com.iqser.red.storage.commons.exception.StorageException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; +import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService; +import com.knecon.fforesight.databasetenantcommons.providers.events.TenantSyncEvent; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,7 +32,7 @@ import static com.knecon.fforesight.service.layoutparser.internal.api.queue.Layo @Slf4j @Service @RequiredArgsConstructor -public class SaasMigrationService { +public class SaasMigrationService implements TenantSyncService { private final AutomaticAnalysisJob automaticAnalysisJob; private final FileStatusPersistenceService fileStatusPersistenceService; @@ -45,6 +47,13 @@ public class SaasMigrationService { private final SaasAnnotationIdMigrationService saasAnnotationIdMigrationService; + + @Override + public void syncTenant(TenantSyncEvent tenantSyncEvent) { + + startMigrationForTenant(tenantSyncEvent.getTenantId()); + } + // Persistence-Service needs to be scaled to 1. public void startMigrationForTenant(String tenantId) { @@ -138,4 +147,6 @@ public class SaasMigrationService { } + + } From 5d6ed6bba834d1379091671bc9a380146e82f33c Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 8 Nov 2023 15:06:46 +0200 Subject: [PATCH 3/5] cleanup --- .../src/main/resources/application-dev.yaml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml index 82da3b633..0be5aa24b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml @@ -35,19 +35,6 @@ persistence-service: fforesight: springdoc: auth-server-url: 'http://localhost:8080/auth' - - jobs: enabled: true - datasource: - url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:25432}/${PSQL_DATABASE:tenantmanager}?ApplicationName=${spring.application.name:}-scheduler&cachePrepStmts=true&useServerPrepStmts=true&rewriteBatchedStatements=true - driverClassName: org.postgresql.Driver - username: ${PSQL_USERNAME:tenantmanager} - password: ${PSQL_PASSWORD:r3dact3d} - platform: org.hibernate.dialect.PostgreSQL95Dialect - hikari: - maximumPoolSize: 2 - data-source-properties: - cachePrepStmts: true - prepStmtCacheSize: 1000 - prepStmtCacheSqlLimit: 2048 + From e73b6bd69bf1e35e376438f0f435d7e62f7faad8 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 23 Nov 2023 21:38:09 +0200 Subject: [PATCH 4/5] Migration local testing --- .../configuration/UserMessagingConfiguration.java | 2 +- .../processor/migration/SaasMigrationService.java | 4 ++-- .../src/main/resources/application-dev.yaml | 14 +++++++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/UserMessagingConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/UserMessagingConfiguration.java index a92085370..e3f399e5f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/UserMessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/UserMessagingConfiguration.java @@ -141,7 +141,7 @@ public class UserMessagingConfiguration { @Bean(name = "userExchange") - TopicExchange tenantExchange(@Value("${fforesight.user-exchange.name}") String userExchange) { + TopicExchange userExchange(@Value("${fforesight.user-exchange.name}") String userExchange) { return new TopicExchange(userExchange); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index cb023ab9a..302d5ecf3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -99,8 +99,8 @@ public class SaasMigrationService implements TenantSyncService { public void handleError(String dossierId, String fileId, String errorCause, String retryQueue) { var migrationEntry = saasMigrationStatusPersistenceService.findById(fileId); - int numErrors = migrationEntry.getProcessingErrorCounter(); - if (numErrors <= settings.getMaxErrorRetries()) { + Integer numErrors = migrationEntry.getProcessingErrorCounter(); + if (numErrors !=null && numErrors <= settings.getMaxErrorRetries()) { saasMigrationStatusPersistenceService.updateErrorCounter(fileId, numErrors + 1, errorCause); rabbitTemplate.convertAndSend(retryQueue, MigrationRequest.builder() .dossierId(dossierId) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml index 0be5aa24b..d4007e382 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml @@ -22,6 +22,7 @@ multitenancy: prepStmtCacheSqlLimit: 2048 liquibase: changeLog: classpath:db/changelog/db.changelog-tenant.yaml + clear-checksums: true monitoring:enabled: true cors.enabled: true @@ -37,4 +38,15 @@ fforesight: auth-server-url: 'http://localhost:8080/auth' jobs: enabled: true - + datasource: + url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:25432}/${PSQL_DATABASE:tenantmanager}?ApplicationName=${spring.application.name:}-scheduler&cachePrepStmts=true&useServerPrepStmts=true&rewriteBatchedStatements=true + driverClassName: org.postgresql.Driver + username: ${PSQL_USERNAME:tenantmanager} + password: ${PSQL_PASSWORD:r3dact3d} + platform: org.hibernate.dialect.PostgreSQL95Dialect + hikari: + maximumPoolSize: 2 + data-source-properties: + cachePrepStmts: true + prepStmtCacheSize: 1000 + prepStmtCacheSqlLimit: 2048 From 1c2d3fe4e21d70653a1f6a67078b37cfa81e6370 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 23 Nov 2023 21:46:40 +0200 Subject: [PATCH 5/5] skipping bugs since this was disabled up to now but got enabled due to upgrade of dependencies --- persistence-service-v1/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index 3256096ac..8cae651b1 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -31,6 +31,8 @@ + true + true 4.177.0 2.71.0 4.38.0