Merge branch 'RED-1137' into 'master'

RED-1137: Enabled tracing, enabled traceId/spanId in Json logging, send custom... Closes RED-1137 See merge request redactmanager/persistence-service!256
2023-12-08 13:46:41 +01:00 · 2023-12-08 13:46:41 +01:00 · ea81708227
commit ea81708227
parent 6f5f148731 c1540943a4
8 changed files with 152 additions and 51 deletions
--- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
+++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
@ -3,14 +3,14 @@ plugins {
    id("io.freefair.lombok") version "8.4"
 }

-val springBootStarterVersion = "3.1.3"
+val springBootStarterVersion = "3.1.5"

 dependencies {
    api(project(":persistence-service-external-api-v1"))
    api(project(":persistence-service-internal-api-v1"))
    api("com.knecon.fforesight:jobs-commons:0.10.0")
-    api("com.knecon.fforesight:database-tenant-commons:0.16.0")
-    api("com.knecon.fforesight:keycloak-commons:0.18.0")
+    api("com.knecon.fforesight:database-tenant-commons:0.17.0")
+    api("com.knecon.fforesight:keycloak-commons:0.22.0")
    api("com.knecon.fforesight:swagger-commons:0.5.0")
    api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0")
    api("com.iqser.red.service:redaction-service-api-v1:4.177.0")
@ -24,6 +24,7 @@ dependencies {
    api("org.springframework.boot:spring-boot-starter-data-jpa:${springBootStarterVersion}")
    api("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
    api("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
+    api("org.springframework.boot:spring-boot-starter-web:${springBootStarterVersion}")
    api("com.iqser.red.commons:spring-commons:2.1.0")
    api("com.iqser.red.commons:jackson-commons:2.1.0")
    api("org.apache.commons:commons-compress:1.21")
@ -31,7 +32,6 @@ dependencies {
    api("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2")
    api("org.postgresql:postgresql:42.2.23")
    api("org.apache.commons:commons-lang3:3.12.0")
-    api("org.springframework.boot:spring-boot-starter-web:3.1.3")
    api("com.iqser.red.commons:spring-boot-starter-web-custom-commons:2.1.0")
    api("com.iqser.red.commons:metric-commons:2.1.0")
    api("com.opencsv:opencsv:5.4")
--- a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts
+++ b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts
@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
    application
    id("com.iqser.red.service.java-conventions")
-    id("org.springframework.boot") version "3.1.3"
+    id("org.springframework.boot") version "3.1.5"
    id("io.spring.dependency-management") version "1.1.3"
    id("org.sonarqube") version "4.4.1.3373"
    id("io.freefair.lombok") version "8.4"
@ -11,22 +11,25 @@ plugins {

 configurations {
    all {
-        exclude(group = "org.springframework.boot", module = "spring-boot-starter-logging")
+        exclude(group = "commons-logging", module = "commons-logging")
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
+        exclude(group = "com.iqser.red.commons", module = "logging-commons")
    }
 }

 dependencies {
    api(project(":persistence-service-processor-v1"))
-    api("com.iqser.red.commons:storage-commons:2.45.0")
-    api("junit:junit:4.13.2")
    api(project(":persistence-service-external-api-impl-v1"))
    api(project(":persistence-service-external-api-impl-v2"))
    api(project(":persistence-service-internal-api-impl-v1"))
+    api("com.iqser.red.commons:storage-commons:2.45.0")
+    api("junit:junit:4.13.2")
+    api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
+    api("net.logstash.logback:logstash-logback-encoder:7.4")
    testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
    testImplementation("org.springframework.security:spring-security-test:6.0.2")
    testImplementation("org.testcontainers:postgresql:1.17.1")
    testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
-    testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
    testImplementation("com.yannbriancon:spring-hibernate-query-utils:2.0.0")
 }

--- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
+++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
@ -1,5 +1,36 @@
 package com.iqser.red.service.peristence.v1.server;

+import java.util.Map;
+import java.util.Optional;
+
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.tracing.ConditionalOnEnabledTracing;
+import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Import;
+import org.springframework.http.server.observation.DefaultServerRequestObservationConvention;
+import org.springframework.http.server.observation.ServerRequestObservationContext;
+import org.springframework.http.server.observation.ServerRequestObservationConvention;
+import org.springframework.retry.annotation.EnableRetry;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.web.servlet.HandlerMapping;
+import org.springframework.web.servlet.config.annotation.CorsRegistry;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
 import com.iqser.red.persistence.service.v1.external.api.impl.PersistenceServiceExternalApiConfiguration;
 import com.iqser.red.persistence.service.v2.external.api.impl.PersistenceServiceExternalApiConfigurationV2;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
@ -17,27 +48,13 @@ import com.knecon.fforesight.tenantcommons.AsyncConfig;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyMessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyWebConfiguration;
+
+import io.micrometer.common.KeyValue;
+import io.micrometer.common.KeyValues;
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.observation.ObservationRegistry;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
-import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
-import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
-import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.cache.annotation.EnableCaching;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Import;
-import org.springframework.retry.annotation.EnableRetry;
-import org.springframework.scheduling.annotation.EnableAsync;
-import org.springframework.scheduling.annotation.EnableScheduling;
-import org.springframework.web.servlet.config.annotation.CorsRegistry;
-import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

@Slf4j
@EnableAsync
@ -48,7 +65,7 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, CassandraAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class})
@Import({PersistenceServiceExternalApiConfigurationV2.class, PersistenceServiceExternalApiConfiguration.class, PersistenceServiceInternalApiConfiguration.class, PersistenceServiceExternalApiCacheConfiguration.class, MultiTenancyWebConfiguration.class, PersistenceServiceProcessorConfiguration.class, MessagingConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class})
-public class Application {
+public class Application implements ApplicationContextAware {

    /**
     * Entry point to the service application.
@ -91,4 +108,71 @@ public class Application {
        return new DictionaryMergeService();
    }

+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+
+        var beanFactory = applicationContext.getAutowireCapableBeanFactory();
+
+        try {
+            var observationRegistry = beanFactory.getBean(ObservationRegistry.class);
+
+            beanFactory.initializeBean(observationRegistry, "observationRegistry");
+        } catch (Exception ignored) {
+        }
+    }
+
+
+    @Bean
+    @ConditionalOnEnabledTracing
+    public ServerRequestObservationConvention customTagsRequestObservationConvention(@Value("${kubernetes.namespace}") String namespace,
+                                                                                     @Value("${project.version}") String version) {
+
+        return new DefaultServerRequestObservationConvention() {
+            @Override
+            public KeyValues getLowCardinalityKeyValues(ServerRequestObservationContext context) {
+                // Make sure that KeyValues entries are already sorted by name for better performance
+                return super.getLowCardinalityKeyValues(context)
+                        .and(getValueFromPathVariableOrRequestParam(context, "dossierId"),
+                                getValueFromPathVariableOrRequestParam(context, "dossierTemplateId"),
+                                getValueFromPathVariableOrRequestParam(context, "fileId"),
+                                KeyValue.of("kubernetes.namespace", namespace),
+                                KeyValue.of("service.version", version),
+                                tenantId(context));
+            }
+
+
+            @Override
+            public KeyValues getHighCardinalityKeyValues(ServerRequestObservationContext context) {
+                // Make sure that KeyValues entries are already sorted by name for better performance
+                return super.getHighCardinalityKeyValues(context)
+                        .and(getValueFromPathVariableOrRequestParam(context, "dossierId"),
+                                getValueFromPathVariableOrRequestParam(context, "dossierTemplateId"),
+                                getValueFromPathVariableOrRequestParam(context, "fileId"),
+                                KeyValue.of("kubernetes.namespace", namespace),
+                                KeyValue.of("service.version", version),
+                                tenantId(context));
+            }
+
+
+            private KeyValue tenantId(ServerRequestObservationContext context) {
+
+                return KeyValue.of("tenantId", Optional.ofNullable(context.getCarrier().getHeader("X-Tenant-Id")).orElse(""));
+            }
+
+
+            private KeyValue getValueFromPathVariableOrRequestParam(ServerRequestObservationContext context, String name) {
+
+                var value = context.getCarrier().getParameter(name);
+                if (value == null) {
+                    var pathVariables = (Map) context.getCarrier().getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE);
+                    value = pathVariables == null ? null : (String) pathVariables.get(name);
+                }
+
+                return KeyValue.of(name, Optional.ofNullable(value).orElse(""));
+            }
+
+        };
+    }
+
 }
--- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml
+++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yaml
@ -50,3 +50,7 @@ fforesight:
          cachePrepStmts: true
          prepStmtCacheSize: 1000
          prepStmtCacheSqlLimit: 2048
+
+management:
+  tracing:
+    enabled: false
--- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml
+++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml
@ -7,6 +7,12 @@ redaction-report-service.url: "http://redaction-report-service-v1:8080"
 search-service.url: "http://search-service-v1:8080"
 tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"

+logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
+
+logging.type : ${LOGGING_TYPE:CONSOLE}
+kubernetes.namespace: ${NAMESPACE:default}
+project.version: 1.0-SNAPSHOT
+
 application:
  type: "RedactManager"
  rss.component-log.enabled: false
@ -63,6 +69,14 @@ management:
  health:
    db:
      enabled: false
+  tracing:
+    enabled: ${TRACING_ENABLED:false}
+    sampling:
+      probability: ${TRACING_PROBABILITY:1.0}
+  otlp:
+    tracing:
+      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}
+

 metrics:
  persistence:
--- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/log4j2.xml
+++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/log4j2.xml
@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="INFO">
-    <Properties>
-        <property name="logType">${env:LOGGING_TYPE}</property>
-    </Properties>
-    <Appenders>
-        <Console name="json" target="SYSTEM_OUT">
-            <JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true"
-                        includeTimeMillis="true">
-
-                <KeyValuePair key="version" value="${project.version}"/>
-            </JSONLayout>
-        </Console>
-        <Console name="console" target="SYSTEM_OUT">
-            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n " />
-        </Console>
-    </Appenders>
-    <Loggers>
-        <Root level="info">
-            <AppenderRef ref="${logType}"/>
-        </Root>
-    </Loggers>
-</Configuration>
--- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/logback-spring.xml
+++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/logback-spring.xml
@ -0,0 +1,17 @@
+<configuration>
+
+    <springProperty scope="configuration" name="logType" source="logging.type"/>
+    <springProperty scope="context" name="application.name" source="spring.application.name"/>
+    <springProperty scope="context" name="version" source="project.version"/>
+    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="${logType}"/>
+    </root>
+
+</configuration>
--- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml
+++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yaml
@ -65,6 +65,8 @@ metrics:
    enabled: false

 management:
+  tracing:
+    enabled: false
  endpoint:
    metrics.enabled: true
    health.enabled: true