Merge branch 'RED-5223' into 'master'
RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs Closes RED-5223 See merge request redactmanager/search-service!16
This commit is contained in:
Dominique Eifländer
commit
c877eea3cc
15
publish-custom-image.sh
Executable file
15
publish-custom-image.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
dir=${PWD##*/}
|
||||
gradle assemble
|
||||
|
||||
# Get the current Git branch
|
||||
branch=$(git rev-parse --abbrev-ref HEAD)
|
||||
|
||||
# Get the short commit hash (first 5 characters)
|
||||
commit_hash=$(git rev-parse --short=5 HEAD)
|
||||
|
||||
# Combine branch and commit hash
|
||||
buildName="${USER}-${branch}-${commit_hash}"
|
||||
|
||||
gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
|
||||
echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"
|
||||
@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
|
||||
plugins {
|
||||
application
|
||||
id("com.iqser.red.service.java-conventions")
|
||||
id("org.springframework.boot") version "3.0.4"
|
||||
id("org.springframework.boot") version "3.1.5"
|
||||
id("io.spring.dependency-management") version "1.1.3"
|
||||
id("org.sonarqube") version "4.4.1.3373"
|
||||
id("io.freefair.lombok") version "8.4"
|
||||
@ -13,28 +13,34 @@ description = "search-service-server-v1"
|
||||
|
||||
configurations {
|
||||
all {
|
||||
exclude(group = "org.springframework.boot", module = "spring-boot-starter-logging")
|
||||
exclude(group = "commons-logging", module = "commons-logging")
|
||||
exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
|
||||
exclude(group = "com.iqser.red.commons", module = "logging-commons")
|
||||
}
|
||||
}
|
||||
|
||||
val springBootStarterVersion = "3.1.5"
|
||||
|
||||
dependencies {
|
||||
api("com.knecon.fforesight:tenant-commons:0.13.0")
|
||||
api("com.knecon.fforesight:tenant-commons:0.14.0")
|
||||
api("com.knecon.fforesight:tracing:0.5.0")
|
||||
api("com.google.guava:guava:31.1-jre")
|
||||
api("com.iqser.red.commons:storage-commons:2.27.0")
|
||||
api("com.iqser.red.commons:storage-commons:2.45.0")
|
||||
api(project(":search-service-api-v1"))
|
||||
api("com.iqser.red.service:persistence-service-internal-api-v1:2.93.0")
|
||||
api("com.iqser.red.commons:spring-commons:2.1.0")
|
||||
api("com.iqser.red.commons:logging-commons:2.1.0")
|
||||
api("com.iqser.red.commons:metric-commons:2.1.0")
|
||||
api("com.iqser.red.commons:jackson-commons:2.1.0")
|
||||
api("co.elastic.clients:elasticsearch-java:8.6.2")
|
||||
api("org.opensearch.client:opensearch-rest-client:2.6.0")
|
||||
api("org.opensearch.client:opensearch-java:2.3.0")
|
||||
api("jakarta.json:jakarta.json-api:2.1.1")
|
||||
api("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.1")
|
||||
api("org.springframework.boot:spring-boot-starter-aop:3.0.4")
|
||||
api("org.springframework.boot:spring-boot-starter-amqp:3.0.4")
|
||||
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
|
||||
api("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
|
||||
api("org.springframework.boot:spring-boot-starter-aop:${springBootStarterVersion}")
|
||||
api("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
|
||||
api("net.logstash.logback:logstash-logback-encoder:7.4")
|
||||
api("ch.qos.logback:logback-classic")
|
||||
testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
|
||||
testImplementation("com.iqser.red.commons:test-commons:2.1.0")
|
||||
testImplementation("org.testcontainers:elasticsearch:1.17.6")
|
||||
testImplementation("org.opensearch:opensearch-testcontainers:2.0.0")
|
||||
|
||||
@ -5,10 +5,19 @@ persistence-service.url: "http://persistence-service-v1:8080"
|
||||
tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
|
||||
fforesight.tenants.remote: true
|
||||
|
||||
logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
|
||||
|
||||
logging.type: ${LOGGING_TYPE:CONSOLE}
|
||||
kubernetes.namespace: ${NAMESPACE:default}
|
||||
project.version: 1.0-SNAPSHOT
|
||||
|
||||
|
||||
server:
|
||||
port: 8080
|
||||
|
||||
spring:
|
||||
application:
|
||||
name: search-service
|
||||
main:
|
||||
allow-circular-references: true # FIXME
|
||||
profiles:
|
||||
@ -35,9 +44,11 @@ management:
|
||||
health.enabled: true
|
||||
endpoints.web.exposure.include: prometheus, health
|
||||
metrics.export.prometheus.enabled: ${monitoring.enabled:false}
|
||||
tracing:
|
||||
enabled: ${TRACING_ENABLED:false}
|
||||
sampling:
|
||||
probability: ${TRACING_PROBABILITY:1.0}
|
||||
otlp:
|
||||
tracing:
|
||||
endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}
|
||||
|
||||
search:
|
||||
backend: elasticsearch
|
||||
|
||||
storage:
|
||||
backend: 's3'
|
||||
@ -1,23 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Configuration status="INFO">
|
||||
<Properties>
|
||||
<property name="logType">${env:LOGGING_TYPE}</property>
|
||||
</Properties>
|
||||
<Appenders>
|
||||
<Console name="json" target="SYSTEM_OUT">
|
||||
<JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true"
|
||||
includeTimeMillis="true">
|
||||
|
||||
<KeyValuePair key="version" value="${project.version}"/>
|
||||
</JSONLayout>
|
||||
</Console>
|
||||
<Console name="console" target="SYSTEM_OUT">
|
||||
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n " />
|
||||
</Console>
|
||||
</Appenders>
|
||||
<Loggers>
|
||||
<Root level="info">
|
||||
<AppenderRef ref="${logType}"/>
|
||||
</Root>
|
||||
</Loggers>
|
||||
</Configuration>
|
||||
@ -0,0 +1,17 @@
|
||||
<configuration>
|
||||
|
||||
<springProperty scope="configuration" name="logType" source="logging.type"/>
|
||||
<springProperty scope="context" name="application.name" source="spring.application.name"/>
|
||||
<springProperty scope="context" name="version" source="project.version"/>
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
|
||||
<appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
|
||||
</appender>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="${logType}"/>
|
||||
</root>
|
||||
|
||||
</configuration>
|
||||
@ -26,9 +26,11 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
import org.testcontainers.elasticsearch.ElasticsearchContainer;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.search.v1.server.Application;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
import com.knecon.fforesight.tenantcommons.TenantsClient;
|
||||
import com.knecon.fforesight.tenantcommons.model.SearchConnection;
|
||||
@ -58,7 +60,14 @@ public abstract class AbstractElasticsearchIntegrationTest {
|
||||
|
||||
TenantContext.setTenantId("redaction");
|
||||
when(tenantsClient.getTenant("redaction")).thenReturn(TenantResponse.builder()
|
||||
.searchConnection(SearchConnection.builder().hosts(Set.of("localhost")).port(port).scheme("http").numberOfShards("1").numberOfReplicas("5").indexPrefix("indexprefix").build())
|
||||
.searchConnection(SearchConnection.builder()
|
||||
.hosts(Set.of("localhost"))
|
||||
.port(port)
|
||||
.scheme("http")
|
||||
.numberOfShards("1")
|
||||
.numberOfReplicas("5")
|
||||
.indexPrefix("indexprefix")
|
||||
.build())
|
||||
.build());
|
||||
}
|
||||
|
||||
@ -82,14 +91,14 @@ public abstract class AbstractElasticsearchIntegrationTest {
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||
@ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
|
||||
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
|
||||
public static class TestConfiguration {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public StorageService inmemoryStorage() {
|
||||
public StorageService inmemoryStorage(ObjectMapper objectMapper) {
|
||||
|
||||
return new FileSystemBackedStorageService();
|
||||
return new FileSystemBackedStorageService(objectMapper);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -25,9 +25,11 @@ import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.search.v1.server.Application;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
import com.knecon.fforesight.tenantcommons.TenantsClient;
|
||||
import com.knecon.fforesight.tenantcommons.model.SearchConnection;
|
||||
@ -56,7 +58,14 @@ public abstract class AbstractOpensearchIntegrationTest {
|
||||
|
||||
TenantContext.setTenantId("redaction");
|
||||
when(tenantsClient.getTenant("redaction")).thenReturn(TenantResponse.builder()
|
||||
.searchConnection(SearchConnection.builder().hosts(Set.of("localhost")).port(port).scheme("http").numberOfShards("1").numberOfReplicas("5").indexPrefix("indexprefix").build())
|
||||
.searchConnection(SearchConnection.builder()
|
||||
.hosts(Set.of("localhost"))
|
||||
.port(port)
|
||||
.scheme("http")
|
||||
.numberOfShards("1")
|
||||
.numberOfReplicas("5")
|
||||
.indexPrefix("indexprefix")
|
||||
.build())
|
||||
.build());
|
||||
}
|
||||
|
||||
@ -80,14 +89,14 @@ public abstract class AbstractOpensearchIntegrationTest {
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||
@ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
|
||||
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
|
||||
public static class TestConfiguration {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public StorageService inmemoryStorage() {
|
||||
public StorageService inmemoryStorage(ObjectMapper objectMapper) {
|
||||
|
||||
return new FileSystemBackedStorageService();
|
||||
return new FileSystemBackedStorageService(objectMapper);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,121 +0,0 @@
|
||||
package com.iqser.red.service.search.v1.server.service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
public class FileSystemBackedStorageService implements StorageService {
|
||||
|
||||
private final Map<String, File> dataMap = new HashMap<>();
|
||||
|
||||
|
||||
public FileSystemBackedStorageService() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public void downloadTo(String tenantId, String objectId, File destinationFile) {
|
||||
|
||||
var res = dataMap.get(objectId);
|
||||
if (res == null) {
|
||||
throw new StorageObjectDoesNotExist(new RuntimeException());
|
||||
}
|
||||
|
||||
IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void deleteObject(String tenantId, String objectId) {
|
||||
|
||||
dataMap.remove(objectId);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean objectExists(String tenantId, String objectId) {
|
||||
|
||||
return dataMap.containsKey(objectId);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public <T> void storeJSONObject(String tenantId, String objectId, T any) {
|
||||
|
||||
File tempFile = File.createTempFile("test", ".tmp");
|
||||
getMapper().writeValue(new FileOutputStream(tempFile), any);
|
||||
dataMap.put(objectId, tempFile);
|
||||
}
|
||||
|
||||
|
||||
private ObjectMapper getMapper() {
|
||||
|
||||
return ObjectMapperFactory.create();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
|
||||
|
||||
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
|
||||
throw new StorageObjectDoesNotExist("Stored object not found");
|
||||
}
|
||||
return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
|
||||
}
|
||||
|
||||
|
||||
public List<String> listPaths() {
|
||||
|
||||
return new ArrayList<>(dataMap.keySet());
|
||||
}
|
||||
|
||||
|
||||
public List<String> listFilePaths() {
|
||||
|
||||
return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void storeObject(String tenantId, String objectId, InputStream stream) {
|
||||
|
||||
File tempFile = File.createTempFile("test", ".tmp");
|
||||
|
||||
try (var fileOutputStream = new FileOutputStream(tempFile)) {
|
||||
IOUtils.copy(stream, fileOutputStream);
|
||||
}
|
||||
|
||||
dataMap.put(objectId, tempFile);
|
||||
}
|
||||
|
||||
|
||||
public void clearStorage() {
|
||||
|
||||
this.dataMap.forEach((k, v) -> {
|
||||
v.delete();
|
||||
});
|
||||
this.dataMap.clear();
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user