Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs

Browse Source
This commit is contained in:
Dominique Eifländer 2023-12-11 17:24:51 +01:00
parent fe8e4c0015
commit b424fd9359
11 changed files with 102 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
publish-custom-docker-img.sh
View File

@ -1,27 +1,15 @@
#!/bin/bash
# This script compiles the project, builds a docker image with the tag <BranchName>-<CommitHash> and pushes it to our nexus.
# Set the Nexus repository URL
NEXUS_REPO="nexus.knecon.com:5001"
# Set the image name
IMAGE_NAME="red/redaction-report-service-server-v1"
# path to image repo
IMAGE_REPO="redaction-report-service-image-v1"
echo "Running build"
mvn clean install -Pquickbuild
dir=${PWD##*/}
gradle assemble
# Get the current Git branch
GIT_BRANCH=$(git symbolic-ref --short HEAD)
# Get the first 5 characters of the commit hash
GIT_COMMIT_HASH=$(git rev-parse --short=5 HEAD)
# Create the image tag by combining branch and commit hash
IMAGE_TAG="${GIT_BRANCH}-${GIT_COMMIT_HASH}"
IMAGE_NAME="$NEXUS_REPO/$IMAGE_NAME:$IMAGE_TAG"
branch=$(git rev-parse --abbrev-ref HEAD)
echo "Building docker image: {$IMAGE_NAME}"
# Build the Docker image with the specified name and tag and push to nexus
mvn -f $IMAGE_REPO docker:build docker:push -Ddocker.image.version=$IMAGE_TAG
# Get the short commit hash (first 5 characters)
commit_hash=$(git rev-parse --short=5 HEAD)
echo "Docker image '$IMAGE_NAME' has been built and pushed to Nexus."
# Combine branch and commit hash
buildName="${USER}-${branch}-${commit_hash}"
gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"

32
redaction-report-service-v1/redaction-report-service-server-v1/build.gradle.kts
View File

@ -3,7 +3,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
application
id("com.iqser.red.service.java-conventions")
id("org.springframework.boot") version "3.0.4"
id("org.springframework.boot") version "3.1.5"
id("io.spring.dependency-management") version "1.1.0"
id("org.sonarqube") version "4.2.1.3168"
id("io.freefair.lombok") version "8.1.0"
@ -11,16 +11,19 @@ plugins {
description = "redaction-service-server-v1"
val tenantCommonVersion = "0.10.0"
val tenantCommonVersion = "0.14.0"
val springCommonsVersion = "2.1.0"
val storageCommonsVersion = "2.27.0"
val storageCommonsVersion = "2.45.0"
val poiVersion = "5.2.3"
val metricCommonsVersion = "2.1.0"
val persistenceServiceVersion = "2.239.0"
val springBootStarterVersion = "3.1.5"
configurations {
all {
exclude(group = "org.springframework.boot", module = "spring-boot-starter-logging")
exclude(group = "commons-logging", module = "commons-logging")
exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
exclude(group = "com.iqser.red.commons", module = "logging-commons")
}
}
@ -28,26 +31,29 @@ description = "redaction-report-service-server-v1"
dependencies {
implementation(project(":redaction-report-service-api-v1"))
implementation("com.knecon.fforesight:tenant-commons:${tenantCommonVersion}")
implementation("com.iqser.red:platform-commons-dependency:2.7.0")
implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}")
implementation("com.knecon.fforesight:tenant-commons:${tenantCommonVersion}")
implementation("com.iqser.red.commons:storage-commons:${storageCommonsVersion}")
implementation("com.iqser.red.commons:spring-commons:${springCommonsVersion}")
implementation("com.iqser.red.commons:metric-commons:${metricCommonsVersion}")
implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}")
implementation("com.knecon.fforesight:tracing:0.5.0")
implementation("org.apache.poi:poi:${poiVersion}")
implementation("org.apache.poi:poi-ooxml:${poiVersion}")
implementation("org.apache.poi:poi-scratchpad:${poiVersion}")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.4")
implementation("org.springframework.boot:spring-boot-starter-test:3.0.4") {
exclude("org.springframework.boot:spring-boot-starter-tomcat:3.0.4")
}
implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.1")
implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
implementation("org.apache.commons:commons-lang3:3.12.0")
implementation("net.logstash.logback:logstash-logback-encoder:7.4")
implementation("org.springframework.boot:spring-boot-starter-logging")
implementation("ch.qos.logback:logback-classic")
testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}") {
exclude("org.springframework.boot:spring-boot-starter-tomcat:3.0.4")
}
testImplementation("com.iqser.red.commons:test-commons:2.1.0")
testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.6")
}

2
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/controller/ControllerAdvice.java
View File

@ -8,8 +8,8 @@ import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import com.amazonaws.services.kms.model.NotFoundException;
import com.iqser.red.commons.spring.ErrorMessage;
import com.iqser.red.service.redaction.report.v1.server.exception.NotFoundException;
import lombok.extern.slf4j.Slf4j;

15
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/exception/NotFoundException.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.redaction.report.v1.server.exception;
public class NotFoundException extends RuntimeException {
public NotFoundException(String message) {
super(message);
}
public NotFoundException(String message, Throwable t) {
super(message, t);
}
}

2
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java
View File

@ -21,11 +21,11 @@ import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;
import org.springframework.stereotype.Service;
import com.amazonaws.services.kms.model.NotFoundException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateClient;
import com.iqser.red.service.redaction.report.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;

15
redaction-report-service-v1/redaction-report-service-server-v1/src/main/resources/application.yml
View File

@ -8,7 +8,15 @@ fforesight.tenants.remote: true
server:
port: 8080
logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
logging.type: ${LOGGING_TYPE:CONSOLE}
kubernetes.namespace: ${NAMESPACE:default}
project.version: 1.0-SNAPSHOT
spring:
application:
name: redaction-report-service
main:
allow-circular-references: true # FIXME
profiles:
@ -35,6 +43,13 @@ management:
health.enabled: true
endpoints.web.exposure.include: prometheus, health
metrics.export.prometheus.enabled: ${monitoring.enabled:false}
tracing:
enabled: ${TRACING_ENABLED:false}
sampling:
probability: ${TRACING_PROBABILITY:1.0}
otlp:
tracing:
endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}
storage:
backend: 's3'

23
redaction-report-service-v1/redaction-report-service-server-v1/src/main/resources/log4j2.xml
View File

@ -1,23 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO">
<Properties>
<property name="logType">${env:LOGGING_TYPE}</property>
</Properties>
<Appenders>
<Console name="json" target="SYSTEM_OUT">
<JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true"
includeTimeMillis="true">
<KeyValuePair key="version" value="${project.version}"/>
</JSONLayout>
</Console>
<Console name="console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n " />
</Console>
</Appenders>
<Loggers>
<Root level="info">
<AppenderRef ref="${logType}"/>
</Root>
</Loggers>
</Configuration>

17
redaction-report-service-v1/redaction-report-service-server-v1/src/main/resources/logback-spring.xml Normal file
View File

@ -0,0 +1,17 @@
<configuration>
<springProperty scope="configuration" name="logType" source="logging.type"/>
<springProperty scope="context" name="application.name" source="spring.application.name"/>
<springProperty scope="context" name="version" source="project.version"/>
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
</appender>
<root level="INFO">
<appender-ref ref="${logType}"/>
</root>
</configuration>

4
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/PlaceholderTest.java
View File

@ -21,7 +21,6 @@ import org.springframework.context.annotation.FilterType;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.amazonaws.services.s3.AmazonS3;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierAttributeType;
@ -53,9 +52,6 @@ public class PlaceholderTest {
@MockBean
private RabbitTemplate rabbitTemplate;
@MockBean
private AmazonS3 s3Client;
@MockBean
private ReportTemplateClient reportTemplateClient;

22
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java
View File

@ -45,7 +45,6 @@ import org.springframework.context.annotation.FilterType;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
@ -105,9 +104,6 @@ public class RedactionReportIntegrationTest {
@MockBean
private FileAttributesConfigClient fileAttributesConfigClient;
@MockBean
private AmazonS3 s3Client;
@Autowired
private WordReportGenerationService wordReportGenerationService;
@ -332,9 +328,7 @@ public class RedactionReportIntegrationTest {
FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build();
EntityLog entityLogSecondFile = objectMapper.readValue(new ClassPathResource("files/entityLogWithManualRedactions.json").getInputStream(), EntityLog.class);
List<ReportRedactionEntry> reportEntriesSecondFile = entityLogConverterService.convertAndSort(entityLogSecondFile,
legalBasisMapping,
new HashMap<>());
List<ReportRedactionEntry> reportEntriesSecondFile = entityLogConverterService.convertAndSort(entityLogSecondFile, legalBasisMapping, new HashMap<>());
ClassPathResource templateResource = new ClassPathResource("templates/Seeds-NewJustificationForm.docx");
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
@ -443,9 +437,7 @@ public class RedactionReportIntegrationTest {
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName", fileModel, excelModel, false);
EntityLog entityLogSecondFile = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
List<ReportRedactionEntry> reportEntriesSecondFile = entityLogConverterService.convertAndSort(entityLogSecondFile,
legalBasisMapping,
mapOfEntityDisplayName);
List<ReportRedactionEntry> reportEntriesSecondFile = entityLogConverterService.convertAndSort(entityLogSecondFile, legalBasisMapping, mapOfEntityDisplayName);
FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build();
excelTemplateReportGenerationService.generateExcelReport(reportEntriesSecondFile,
placeholders,
@ -495,21 +487,25 @@ public class RedactionReportIntegrationTest {
}
}
@Test
@SneakyThrows
public void testScmReport() {
Dossier dossier = prepareDossier();
FileModel fileModel = FileModel.builder().filename("filename")
FileModel fileModel = FileModel.builder()
.filename("filename")
.dossierId(dossier.getDossierId())
.dossierTemplateId(dossier.getDossierTemplateId())
.fileAttributes(Map.of("TestAttribute", "Lorem Ipsum", "1b5ebe26-34f2-4c3b-983d-c0f0a010302c", "402"))
.build();
EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/scm/entityLog.json").getInputStream(), EntityLog.class);
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/scm/legalBasisMapping.json").getInputStream(), new TypeReference<>() {});
List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/scm/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
});
ComponentLog componentLog = objectMapper.readValue(new ClassPathResource("files/scm/componentLog.json").getInputStream(), ComponentLog.class);
List<FileAttributeConfig> fileAttributeConfigs = objectMapper.readValue(new ClassPathResource("files/scm/fileAttributes.json").getInputStream(), new TypeReference<>() {});
List<FileAttributeConfig> fileAttributeConfigs = objectMapper.readValue(new ClassPathResource("files/scm/fileAttributes.json").getInputStream(), new TypeReference<>() {
});
Map<String, String> mapOfEntityDisplayName = createEntityDisplayNames(entityLog);
List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(entityLog, legalBasisMapping, mapOfEntityDisplayName);

29
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java
View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.report.v1.server;
import static com.iqser.red.service.redaction.report.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
@ -21,7 +20,6 @@ import java.util.stream.IntStream;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@ -38,7 +36,6 @@ import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -99,9 +96,6 @@ public class RedactionReportV2IntegrationTest {
@MockBean
private FileAttributesConfigClient fileAttributesConfigClient;
@MockBean
private AmazonS3 s3Client;
@MockBean
private MessagingConfiguration messagingConfiguration;
@ -180,7 +174,7 @@ public class RedactionReportV2IntegrationTest {
var templateIds = new HashSet<String>();
for (var template : templates) {
var templateId = Base64.getEncoder().encodeToString(template.getBytes(StandardCharsets.UTF_8));
var templateId = Base64.getEncoder().encodeToString(template.getBytes(StandardCharsets.UTF_8));
templateIds.add(templateId);
log.info("Assigned template Id {} to file {}", templateId, template);
ReportTemplate testTemplate = new ReportTemplate();
@ -234,11 +228,12 @@ public class RedactionReportV2IntegrationTest {
}
}
@SneakyThrows
private void prepareStorage(int id) {
var entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
fileSystemBackedStorageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId("dossierId", "fileId" + id, FileType.ENTITY_LOG), entityLog);
var entityLog = objectMapper.readValue(new ClassPathResource("files/entityLog.json").getInputStream(), EntityLog.class);
fileSystemBackedStorageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId("dossierId", "fileId" + id, FileType.ENTITY_LOG), entityLog);
}
@ -288,14 +283,16 @@ public class RedactionReportV2IntegrationTest {
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_getReportEntries", 1, null);
}
@Test
@SneakyThrows
public void testIncludeUnprocessed() {
var reportRequestMessage = prepareFlow(2, "templates/Excel QC (incl. Skipped Redactions).xlsx");
reportRequestMessage.setIncludeUnprocessed(true);
when(entityLogClient.getEntityLog(anyString(), anyString(), any(), eq(true))).thenReturn(new EntityLog(
1, 1, List.of(EntityLogEntry.builder()
when(entityLogClient.getEntityLog(anyString(), anyString(), any(), eq(true))).thenReturn(new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.state(EntryState.APPLIED)
.positions(List.of(new Position(1, 1, 1, 1, 1)))
.id("123")
@ -306,8 +303,12 @@ public class RedactionReportV2IntegrationTest {
.endOffset(10)
.color(new float[]{0, 0, 0})
.section("section")
.build()), Collections.emptyList(), 1, 1, 1, 1)
);
.build()),
Collections.emptyList(),
1,
1,
1,
1));
processRequest(reportRequestMessage, ".xlsx");
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateReports", 1, null);
@ -327,7 +328,7 @@ public class RedactionReportV2IntegrationTest {
@Configuration
@EnableAutoConfiguration(exclude = {/*StorageAutoConfiguration.class, */RabbitAutoConfiguration.class})
@ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
@ComponentScan("com.iqser.red.service.persistence")
public static class TestConfiguration {