Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

Pull request #214: RED-4609 - adjusted some metrics, added tests for metrics

Browse Source 
Merge in RED/redaction-report-service from RED-4609 to master

* commit '106747c3345146fa3c4518e29f286ccb663eef78':
  RED-4609 - adjusted some metrics, added tests for metrics
This commit is contained in:
Timo Bejan 2023-02-09 12:07:38 +01:00 committed by Dominique Eiflaender
commit f21b0519e7
9 changed files with 147 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java
View File

@ -150,7 +150,6 @@ public class ExcelReportGenerationService {
@SneakyThrows
@Timed("redactmanager_excel-addRows")
private void addRows(SXSSFWorkbook workbook,
Sheet sheet,
Map<CellIdentifier, Cell> copiedCells,
@ -309,7 +308,6 @@ public class ExcelReportGenerationService {
}
@Timed("redactmanager_excel-addRedactionEntryRows")
private void addRedactionEntryRows(Sheet sheet, List<ReportRedactionEntry> reportEntries, String filename, ExcelModel excelModel, PlaceholderModel placeholderModel) {
long start = System.currentTimeMillis();

55
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/RedactionLogConverterService.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -12,16 +13,70 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.report.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierClient;
import com.iqser.red.service.redaction.report.v1.server.client.RedactionLogClient;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class RedactionLogConverterService {
private final RedactionLogClient redactionLogClient;
private final DossierClient dossierClient;
private final DictionaryClient dictionaryClient;
@Timed("redactmanager_getReportEntries")
public List<ReportRedactionEntry> getReportEntries(String dossierId, String fileId, boolean isExcluded) {
if (isExcluded) {
return new ArrayList<>();
}
RedactionLog redactionLog;
Map<String, String> mapOfEntityDisplayName;
try {
redactionLog = redactionLogClient.getRedactionLog(dossierId, fileId, new ArrayList<>(), true, false);
mapOfEntityDisplayName = fillMapOfTypeAndEntityDisplayName(dossierId);
} catch (StorageObjectDoesNotExist e) {
return new ArrayList<>();
}
var legalBasisMappings = redactionLog.getLegalBasis();
return convertAndSort(redactionLog, legalBasisMappings, mapOfEntityDisplayName);
}
private Map<String, String> fillMapOfTypeAndEntityDisplayName(String dossierId) {
List<Type> typeList = new ArrayList<>();
typeList.addAll(dictionaryClient.getAllTypesForDossier(dossierId, false));
typeList.addAll(dictionaryClient.getAllTypesForDossierTemplate(dossierClient.getDossierById(dossierId, true, false).getDossierTemplateId(), false));
Map<String, String> mapOfEntityDisplayName = new HashMap<>();
for (var type : typeList) {
mapOfEntityDisplayName.put(type.getType(), type.getLabel());
}
mapOfEntityDisplayName.put("manual", "Manual");
return mapOfEntityDisplayName;
}
public List<ReportRedactionEntry> convertAndSort(RedactionLog redactionLog, List<RedactionLogLegalBasis> legalBasisMappings, Map<String, String> mapOfEntityDisplayName) {
List<ReportRedactionEntry> reportEntries = new ArrayList<>();

37
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java
View File

@ -56,10 +56,8 @@ public class ReportGenerationService {
private final FileStatusClient fileStatusClient;
private final DossierClient dossierClient;
private final ReportTemplateClient reportTemplateClient;
private final RedactionLogClient redactionLogClient;
private final ExcelReportGenerationService excelTemplateReportGenerationService;
private final GeneratePlaceholderService generatePlaceholderService;
private final DictionaryClient dictionaryClient;
private final ReportTemplateSettings reportTemplateSettings;
private final RSSService rSSService;
@ -90,7 +88,7 @@ public class ReportGenerationService {
var fileStatus = fileStatusClient.getFileStatus(dossierId, fileId);
generatePlaceholderService.resolveFileAttributeValues(fileStatus, placeholderModel);
List<ReportRedactionEntry> reportEntries = getReportEntries(dossierId, fileId, fileStatus.isExcluded());
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.getReportEntries(dossierId, fileId, fileStatus.isExcluded());
generateMultiFileExcelReports(reportTemplates.multiFileWorkbookReportTemplates, placeholderModel, fileStatus, isLastFile, dossierName, reportEntries);
@ -301,42 +299,9 @@ public class ReportGenerationService {
}
@Timed("redactmanager_getReportEntries")
private List<ReportRedactionEntry> getReportEntries(String dossierId, String fileId, boolean isExcluded) {
if (isExcluded) {
return new ArrayList<>();
}
RedactionLog redactionLog;
Map<String, String> mapOfEntityDisplayName;
try {
redactionLog = redactionLogClient.getRedactionLog(dossierId, fileId, new ArrayList<>(), true, false);
mapOfEntityDisplayName = fillMapOfTypeAndEntityDisplayName(dossierId);
} catch (StorageObjectDoesNotExist e) {
return new ArrayList<>();
}
var legalBasisMappings = redactionLog.getLegalBasis();
return redactionLogConverterService.convertAndSort(redactionLog, legalBasisMappings, mapOfEntityDisplayName);
}
private Map<String, String> fillMapOfTypeAndEntityDisplayName(String dossierId) {
List<Type> typeList = new ArrayList<>();
typeList.addAll(dictionaryClient.getAllTypesForDossier(dossierId, false));
typeList.addAll(dictionaryClient.getAllTypesForDossierTemplate(dossierClient.getDossierById(dossierId, true, false).getDossierTemplateId(), false));
Map<String, String> mapOfEntityDisplayName = new HashMap<>();
for (var type : typeList) {
mapOfEntityDisplayName.put(type.getType(), type.getLabel());
}
mapOfEntityDisplayName.put("manual", "Manual");
return mapOfEntityDisplayName;
}
@FieldDefaults(makeFinal = true, level = AccessLevel.PUBLIC)

5
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java
View File

@ -222,8 +222,7 @@ public class WordReportGenerationService {
}
@Timed("redactmanager_word-replaceTextPlaceholders")
public int replaceTextPlaceholders(XWPFDocument doc,
private int replaceTextPlaceholders(XWPFDocument doc,
PlaceholderModel placeholderModel,
String dossierName,
String fileName,
@ -335,7 +334,6 @@ public class WordReportGenerationService {
}
@Timed("redactmanager_word-computePlaceholderPos")
private PlaceHolderFunctions computePlaceholderPos(XWPFTable table, PlaceholderModel placeholderModel) {
Set<String> foundPlaceHolder = new HashSet<>();
@ -357,7 +355,6 @@ public class WordReportGenerationService {
}
@Timed("redactmanager_word-addRedactionEntryRows")
private int addRedactionEntryRows(XWPFTable table,
List<ReportRedactionEntry> reportEntries,
String filename,

10
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/PlaceholderTest.java
View File

@ -11,7 +11,10 @@ import org.apache.commons.io.IOUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
@ -27,10 +30,12 @@ import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesCon
import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateClient;
import com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = RANDOM_PORT)
@EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class PlaceholderTest {
@Autowired
@ -39,6 +44,9 @@ public class PlaceholderTest {
@MockBean
private StorageService storageService;
@MockBean
private RabbitTemplate rabbitTemplate;
@MockBean
private AmazonS3 s3Client;

14
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java
View File

@ -20,15 +20,21 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.junit.After;
import org.junit.Test;
import org.junit.jupiter.api.AfterEach;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
@ -56,12 +62,15 @@ import com.iqser.red.service.redaction.report.v1.server.service.WordReportGenera
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import lombok.SneakyThrows;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = RANDOM_PORT)
@EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class RedactionReportIntegrationTest {
@Autowired
@ -70,6 +79,9 @@ public class RedactionReportIntegrationTest {
@MockBean
private StorageService storageService;
@MockBean
private RabbitTemplate rabbitTemplate;
@MockBean
private ReportStorageService reportStorageService;

25
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java
View File

@ -9,9 +9,12 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.junit.After;
import org.junit.Test;
import org.junit.jupiter.api.AfterEach;
import org.junit.runner.RunWith;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
@ -52,10 +55,12 @@ import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateCli
import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.redaction.report.v1.server.service.ReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.utils.FileSystemBackedStorageService;
import com.iqser.red.service.redaction.report.v1.server.utils.MetricValidationUtils;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -107,6 +112,9 @@ public class RedactionReportV2IntegrationTest {
@Autowired
private FileSystemBackedStorageService fileSystemBackedStorageService;
@Autowired
private PrometheusMeterRegistry prometheusMeterRegistry;
@SneakyThrows
private ReportRequestMessage prepareFlow(int numOfFiles, String... templates) {
@ -210,6 +218,11 @@ public class RedactionReportV2IntegrationTest {
var reportRequestMessage = prepareFlow(1, "templates/Excel Report.xlsx");
processRequest(reportRequestMessage, ".xlsx");
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateReports", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateExcelReport", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_calculateExcelModel", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_getReportEntries", 1, null);
}
@ -219,6 +232,13 @@ public class RedactionReportV2IntegrationTest {
var reportRequestMessage = prepareFlow(2, "templates/Justification Appendix A1.docx");
processRequest(reportRequestMessage, ".docx");
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateReports", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateWordReport", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_computeIuclidFunction", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateExcelReport", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_calculateExcelModel", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_getReportEntries", 1, null);
}
@ -228,6 +248,11 @@ public class RedactionReportV2IntegrationTest {
var reportRequestMessage = prepareFlow(1, "templates/report.xlsx", "templates/report-advanced.xlsx");
processRequest(reportRequestMessage, ".xlsx");
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateReports", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_generateExcelReport", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_calculateExcelModel", 1, null);
MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_getReportEntries", 1, null);
}

29
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/utils/MetricValidationUtils.java Normal file
View File

@ -0,0 +1,29 @@
package com.iqser.red.service.redaction.report.v1.server.utils;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import java.util.concurrent.TimeUnit;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.micrometer.prometheus.PrometheusTimer;
import lombok.experimental.UtilityClass;
@UtilityClass
public class MetricValidationUtils {
public static void validateMetric(PrometheusMeterRegistry registry, String name, int invocationCount, Integer time) {
var metricOptional = registry.getMeters().stream().filter(m -> m.getId().getName().equalsIgnoreCase(name)).findAny();
assertThat(metricOptional.isPresent()).isTrue();
PrometheusTimer metric = (PrometheusTimer) metricOptional.get();
assertThat(metric.count()).isGreaterThanOrEqualTo(invocationCount);
if (time != null) {
// for actually timing the method ...
assertThat(metric.mean(TimeUnit.MILLISECONDS)).isGreaterThan(time);
} else {
// everything takes at least one nano-second - test the meter is correctly initialized
assertThat(metric.mean(TimeUnit.NANOSECONDS)).isGreaterThan(1);
}
}
}

14
redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/application.yml Normal file
View File

@ -0,0 +1,14 @@
management:
endpoint:
metrics.enabled: true
prometheus.enabled: true
health.enabled: true
endpoints.web.exposure.include: prometheus, health, metrics
metrics.export.prometheus.enabled: true
spring:
main:
allow-circular-references: true
persistence-service.url: "http://mock.url"