From 36e76b027387cca4f8556d855d74009a04a8f33c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 14 Dec 2023 11:28:14 +0100 Subject: [PATCH 1/2] RED-5223: Added tracing for scheduled analysis, and add fileId on upload to trace --- .../v1/processor/service/UploadService.java | 12 +++++- .../service/job/AutomaticAnalysisJob.java | 40 ++++++++++++++----- .../peristence/v1/server/Application.java | 13 +----- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadService.java index ad7089ad9..a6f83104f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadService.java @@ -27,6 +27,7 @@ import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; import com.knecon.fforesight.tenantcommons.TenantContext; +import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -41,6 +42,7 @@ public class UploadService { private final AuditPersistenceService auditPersistenceService; private final FileAttributesManagementService fileAttributesManagementService; private final StorageService storageService; + private final ObservationRegistry observationRegistry; @PreAuthorize("hasAuthority('" + WRITE_FILE_ATTRIBUTES + "')") @@ -67,6 +69,8 @@ public class UploadService { dossierManagementService.getDossierById(dossierId, false, false); var fileId = generateFileId(fileName, dossierId); + addFileIdToTrace(fileId); + var storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.UNTOUCHED); try { @@ -88,7 +92,6 @@ public class UploadService { .build()); return FileUploadResult.builder().fileIds(Collections.singletonList(fileId)).build(); - } @@ -97,4 +100,11 @@ public class UploadService { return hashFunction.hashBytes((fileName + dossierId).getBytes(StandardCharsets.UTF_8)).toString(); } + private void addFileIdToTrace(String fileId){ + if(observationRegistry.getCurrentObservation() != null){ + observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId); + } + } + + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index 3abb8fef3..80a04dd77 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -17,8 +17,9 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUti import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantProvider; -import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -32,6 +33,7 @@ public class AutomaticAnalysisJob implements Job { private final FileManagementServiceSettings fileManagementServiceSettings; private final FileStatusService fileStatusService; private final TenantProvider tenantProvider; + private final ObservationRegistry observationRegistry; @Setter private boolean schedulingStopped; @@ -79,13 +81,7 @@ public class AutomaticAnalysisJob implements Job { var next = allStatusesIterator.next(); // in case the file doesn't have numberOfPages set, we assume an average. - if (next.isFullAnalysisRequired()) { - log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), next.getFilename()); - fileStatusService.setStatusFullReprocess(next.getDossierId(), next.getId(), false, false); - } else if (next.isReanalysisRequired()) { - log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), next.getFilename()); - fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false); - } + reanalyseFile(next); queuedFiles++; } @@ -99,17 +95,41 @@ public class AutomaticAnalysisJob implements Job { } + private void reanalyseFile(FileModel file) { + + Observation.createNotStarted("AutomaticAnalysisJob", observationRegistry) + .contextualName("scheduled-analysis") + .highCardinalityKeyValue("dossierTemplateId", file.getDossierTemplateId()) + .highCardinalityKeyValue("dossierId", file.getDossierId()) + .highCardinalityKeyValue("fileId", file.getId()) + .lowCardinalityKeyValue("tenantId", TenantContext.getTenantId()) + .observe(() -> { + + if (file.isFullAnalysisRequired()) { + log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getFilename()); + fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false); + } else if (file.isReanalysisRequired()) { + log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getFilename()); + fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), false); + } + }); + } + + private List getAllRelevantStatuses() { return fileStatusService.getAllRelevantStatusesForReanalysisScheduler(); } - public void stopForTenant(String tenantId){ + public void stopForTenant(String tenantId) { + stoppedTenants.add(tenantId); } - public void startForTenant(String tenantId){ + + public void startForTenant(String tenantId) { + stoppedTenants.remove(tenantId); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java index accf2aff2..9060d42ba 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java @@ -132,13 +132,7 @@ public class Application implements ApplicationContextAware { @Override public KeyValues getLowCardinalityKeyValues(ServerRequestObservationContext context) { // Make sure that KeyValues entries are already sorted by name for better performance - return super.getLowCardinalityKeyValues(context) - .and(getValueFromPathVariableOrRequestParam(context, "dossierId"), - getValueFromPathVariableOrRequestParam(context, "dossierTemplateId"), - getValueFromPathVariableOrRequestParam(context, "fileId"), - KeyValue.of("kubernetes.namespace", namespace), - KeyValue.of("service.version", version), - tenantId(context)); + return super.getLowCardinalityKeyValues(context).and(KeyValue.of("kubernetes.namespace", namespace), KeyValue.of("service.version", version), tenantId(context)); } @@ -148,10 +142,7 @@ public class Application implements ApplicationContextAware { return super.getHighCardinalityKeyValues(context) .and(getValueFromPathVariableOrRequestParam(context, "dossierId"), getValueFromPathVariableOrRequestParam(context, "dossierTemplateId"), - getValueFromPathVariableOrRequestParam(context, "fileId"), - KeyValue.of("kubernetes.namespace", namespace), - KeyValue.of("service.version", version), - tenantId(context)); + getValueFromPathVariableOrRequestParam(context, "fileId")); } -- 2.47.2 From 56038f13b9db9574708e9c29c61fd3b7193783c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 14 Dec 2023 13:50:50 +0100 Subject: [PATCH 2/2] hotfix: Fixed old versions in project, because of cycling dependencies --- .../build.gradle.kts | 20 +++++++++--- .../build.gradle.kts | 20 +++++++++--- .../build.gradle.kts | 1 + .../build.gradle.kts | 32 +++++++++++++++---- 4 files changed, 58 insertions(+), 15 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts b/persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts index fabf4463c..b63a3f9da 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts @@ -4,10 +4,22 @@ plugins { dependencies { api(project(":persistence-service-internal-api-v1")) - api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") - api("com.iqser.red.service:redaction-service-api-v1:4.196.0") - api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") - api("com.iqser.red.service:search-service-api-v1:2.71.0") + api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:redaction-service-api-v1:4.196.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:search-service-api-v1:2.71.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.13.4") api("com.google.guava:guava:31.1-jre") api("org.springframework.boot:spring-boot-starter-security:3.1.3") diff --git a/persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts b/persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts index 5e0b5fa5c..258c94896 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts +++ b/persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts @@ -6,10 +6,22 @@ plugins { dependencies { api(project(":persistence-service-external-api-v1")) api(project(":persistence-service-internal-api-v1")) - api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") - api("com.iqser.red.service:redaction-service-api-v1:4.196.0") - api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") - api("com.iqser.red.service:search-service-api-v1:2.71.0") + api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:redaction-service-api-v1:4.196.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:search-service-api-v1:2.71.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.16.0") api("com.google.guava:guava:31.1-jre") api("org.springframework.boot:spring-boot-starter-security:3.1.3") diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/build.gradle.kts b/persistence-service-v1/persistence-service-internal-api-impl-v1/build.gradle.kts index ed3a1919a..c15072b72 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/build.gradle.kts @@ -5,6 +5,7 @@ plugins { dependencies { api(project(":persistence-service-processor-v1")) + api(project(":persistence-service-external-api-v1")) testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4") } diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 4a63960d0..8641bd30f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -8,18 +8,37 @@ val springBootStarterVersion = "3.1.5" dependencies { api(project(":persistence-service-external-api-v1")) api(project(":persistence-service-internal-api-v1")) + api(project(":persistence-service-shared-api-v1")) + api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:redaction-service-api-v1:4.196.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.knecon.fforesight:layoutparser-service-internal-api:0.74.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.iqser.red.service:search-service-api-v1:2.71.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } + api("com.knecon.fforesight.service:ocr-service-api:4.5.0") { + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + } api("com.knecon.fforesight:jobs-commons:0.10.0") api("com.knecon.fforesight:database-tenant-commons:0.20.0") api("com.knecon.fforesight:keycloak-commons:0.24.0") api("com.knecon.fforesight:tracing-commons:0.2.0") api("com.knecon.fforesight:swagger-commons:0.5.0") - api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") - api("com.iqser.red.service:redaction-service-api-v1:4.196.0") - api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") - api("com.knecon.fforesight:layoutparser-service-internal-api:0.74.0") - api("com.iqser.red.service:search-service-api-v1:2.71.0") api("com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:0.4.0") - api("com.knecon.fforesight.service:ocr-service-api:4.5.0") api("org.springframework.security:spring-security-acl:6.0.2") api("org.springframework.boot:spring-boot-starter-mail:${springBootStarterVersion}") api("org.springframework.boot:spring-boot-starter-data-jpa:${springBootStarterVersion}") @@ -38,7 +57,6 @@ dependencies { api("com.opencsv:opencsv:5.4") api("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4") api("commons-validator:commons-validator:1.7") - api("com.iqser.red.service:persistence-service-shared-api-v1:+") testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2") testImplementation("org.testcontainers:postgresql:1.17.1") testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4") -- 2.47.2