Merge branch 'RED-5223' into 'master'

RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs

Closes RED-5223

See merge request redactmanager/search-service!16

publish-custom-image.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+dir=${PWD##*/}
+gradle assemble
+
+# Get the current Git branch
+branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Get the short commit hash (first 5 characters)
+commit_hash=$(git rev-parse --short=5 HEAD)
+
+# Combine branch and commit hash
+buildName="${USER}-${branch}-${commit_hash}"
+
+gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
+echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"

search-service-v1/search-service-server-v1/build.gradle.kts

@@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
     application
     id("com.iqser.red.service.java-conventions")
-    id("org.springframework.boot") version "3.0.4"
+    id("org.springframework.boot") version "3.1.5"
     id("io.spring.dependency-management") version "1.1.3"
     id("org.sonarqube") version "4.4.1.3373"
     id("io.freefair.lombok") version "8.4"
@@ -13,28 +13,34 @@ description = "search-service-server-v1"
 
 configurations {
     all {
-        exclude(group = "org.springframework.boot", module = "spring-boot-starter-logging")
+        exclude(group = "commons-logging", module = "commons-logging")
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
+        exclude(group = "com.iqser.red.commons", module = "logging-commons")
     }
 }
 
+val springBootStarterVersion = "3.1.5"
+
 dependencies {
-    api("com.knecon.fforesight:tenant-commons:0.13.0")
+    api("com.knecon.fforesight:tenant-commons:0.14.0")
+    api("com.knecon.fforesight:tracing:0.5.0")
     api("com.google.guava:guava:31.1-jre")
-    api("com.iqser.red.commons:storage-commons:2.27.0")
+    api("com.iqser.red.commons:storage-commons:2.45.0")
     api(project(":search-service-api-v1"))
     api("com.iqser.red.service:persistence-service-internal-api-v1:2.93.0")
     api("com.iqser.red.commons:spring-commons:2.1.0")
-    api("com.iqser.red.commons:logging-commons:2.1.0")
     api("com.iqser.red.commons:metric-commons:2.1.0")
     api("com.iqser.red.commons:jackson-commons:2.1.0")
     api("co.elastic.clients:elasticsearch-java:8.6.2")
     api("org.opensearch.client:opensearch-rest-client:2.6.0")
     api("org.opensearch.client:opensearch-java:2.3.0")
     api("jakarta.json:jakarta.json-api:2.1.1")
-    api("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.1")
+    api("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
-    api("org.springframework.boot:spring-boot-starter-aop:3.0.4")
+    api("org.springframework.boot:spring-boot-starter-aop:${springBootStarterVersion}")
-    api("org.springframework.boot:spring-boot-starter-amqp:3.0.4")
+    api("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
-    testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
+    api("net.logstash.logback:logstash-logback-encoder:7.4")
+    api("ch.qos.logback:logback-classic")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
     testImplementation("com.iqser.red.commons:test-commons:2.1.0")
     testImplementation("org.testcontainers:elasticsearch:1.17.6")
     testImplementation("org.opensearch:opensearch-testcontainers:2.0.0")

search-service-v1/search-service-server-v1/src/main/resources/application.yml

@@ -5,10 +5,19 @@ persistence-service.url: "http://persistence-service-v1:8080"
 tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
 fforesight.tenants.remote: true
 
+logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
+
+logging.type: ${LOGGING_TYPE:CONSOLE}
+kubernetes.namespace: ${NAMESPACE:default}
+project.version: 1.0-SNAPSHOT
+
+
 server:
   port: 8080
 
 spring:
+  application:
+    name: search-service
   main:
     allow-circular-references: true # FIXME
   profiles:
@@ -35,9 +44,11 @@ management:
     health.enabled: true
   endpoints.web.exposure.include: prometheus, health
   metrics.export.prometheus.enabled: ${monitoring.enabled:false}
+  tracing:
+    enabled: ${TRACING_ENABLED:false}
+    sampling:
+      probability: ${TRACING_PROBABILITY:1.0}
+  otlp:
+    tracing:
+      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}
 
-search:
-  backend: elasticsearch
-
-storage:
-  backend: 's3'

search-service-v1/search-service-server-v1/src/main/resources/log4j2.xml

@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="INFO">
-    <Properties>
-        <property name="logType">${env:LOGGING_TYPE}</property>
-    </Properties>
-    <Appenders>
-        <Console name="json" target="SYSTEM_OUT">
-            <JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true"
-                        includeTimeMillis="true">
-
-                <KeyValuePair key="version" value="${project.version}"/>
-            </JSONLayout>
-        </Console>
-        <Console name="console" target="SYSTEM_OUT">
-            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n " />
-        </Console>
-    </Appenders>
-    <Loggers>
-        <Root level="info">
-            <AppenderRef ref="${logType}"/>
-        </Root>
-    </Loggers>
-</Configuration>

search-service-v1/search-service-server-v1/src/main/resources/logback-spring.xml

@@ -0,0 +1,17 @@
+<configuration>
+
+    <springProperty scope="configuration" name="logType" source="logging.type"/>
+    <springProperty scope="context" name="application.name" source="spring.application.name"/>
+    <springProperty scope="context" name="version" source="project.version"/>
+    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="${logType}"/>
+    </root>
+
+</configuration>

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java

@@ -26,9 +26,11 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import org.testcontainers.elasticsearch.ElasticsearchContainer;
 import org.testcontainers.utility.DockerImageName;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.search.v1.server.Application;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 import com.knecon.fforesight.tenantcommons.model.SearchConnection;
@@ -58,7 +60,14 @@ public abstract class AbstractElasticsearchIntegrationTest {
 
         TenantContext.setTenantId("redaction");
         when(tenantsClient.getTenant("redaction")).thenReturn(TenantResponse.builder()
-                .searchConnection(SearchConnection.builder().hosts(Set.of("localhost")).port(port).scheme("http").numberOfShards("1").numberOfReplicas("5").indexPrefix("indexprefix").build())
+                .searchConnection(SearchConnection.builder()
+                        .hosts(Set.of("localhost"))
+                        .port(port)
+                        .scheme("http")
+                        .numberOfShards("1")
+                        .numberOfReplicas("5")
+                        .indexPrefix("indexprefix")
+                        .build())
                 .build());
     }
 
@@ -82,14 +91,14 @@ public abstract class AbstractElasticsearchIntegrationTest {
 
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
     public static class TestConfiguration {
 
         @Bean
         @Primary
-        public StorageService inmemoryStorage() {
+        public StorageService inmemoryStorage(ObjectMapper objectMapper) {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(objectMapper);
         }
 
     }

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java

@@ -25,9 +25,11 @@ import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 import org.testcontainers.utility.DockerImageName;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.search.v1.server.Application;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 import com.knecon.fforesight.tenantcommons.model.SearchConnection;
@@ -56,7 +58,14 @@ public abstract class AbstractOpensearchIntegrationTest {
 
         TenantContext.setTenantId("redaction");
         when(tenantsClient.getTenant("redaction")).thenReturn(TenantResponse.builder()
-                .searchConnection(SearchConnection.builder().hosts(Set.of("localhost")).port(port).scheme("http").numberOfShards("1").numberOfReplicas("5").indexPrefix("indexprefix").build())
+                .searchConnection(SearchConnection.builder()
+                        .hosts(Set.of("localhost"))
+                        .port(port)
+                        .scheme("http")
+                        .numberOfShards("1")
+                        .numberOfReplicas("5")
+                        .indexPrefix("indexprefix")
+                        .build())
                 .build());
     }
 
@@ -80,14 +89,14 @@ public abstract class AbstractOpensearchIntegrationTest {
 
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
     public static class TestConfiguration {
 
         @Bean
         @Primary
-        public StorageService inmemoryStorage() {
+        public StorageService inmemoryStorage(ObjectMapper objectMapper) {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(objectMapper);
         }
 
     }

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java

@@ -1,121 +0,0 @@
-package com.iqser.red.service.search.v1.server.service;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
-public class FileSystemBackedStorageService implements StorageService {
-
-    private final Map<String, File> dataMap = new HashMap<>();
-
-
-    public FileSystemBackedStorageService() {
-
-    }
-
-
-    @SneakyThrows
-    @Override
-    public void downloadTo(String tenantId, String objectId, File destinationFile) {
-
-        var res = dataMap.get(objectId);
-        if (res == null) {
-            throw new StorageObjectDoesNotExist(new RuntimeException());
-        }
-
-        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
-    }
-
-
-    @Override
-    public void deleteObject(String tenantId, String objectId) {
-
-        dataMap.remove(objectId);
-    }
-
-
-    @Override
-    public boolean objectExists(String tenantId, String objectId) {
-
-        return dataMap.containsKey(objectId);
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
-
-        File tempFile = File.createTempFile("test", ".tmp");
-        getMapper().writeValue(new FileOutputStream(tempFile), any);
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    private ObjectMapper getMapper() {
-
-        return ObjectMapperFactory.create();
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
-
-        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
-            throw new StorageObjectDoesNotExist("Stored object not found");
-        }
-        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
-    }
-
-
-    public List<String> listPaths() {
-
-        return new ArrayList<>(dataMap.keySet());
-    }
-
-
-    public List<String> listFilePaths() {
-
-        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
-    }
-
-
-    @Override
-    @SneakyThrows
-    public void storeObject(String tenantId, String objectId, InputStream stream) {
-
-        File tempFile = File.createTempFile("test", ".tmp");
-
-        try (var fileOutputStream = new FileOutputStream(tempFile)) {
-            IOUtils.copy(stream, fileOutputStream);
-        }
-
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    public void clearStorage() {
-
-        this.dataMap.forEach((k, v) -> {
-            v.delete();
-        });
-        this.dataMap.clear();
-    }
-
-}