Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

added metrics

Browse Source
This commit is contained in:
Timo Bejan 2022-02-14 11:16:51 +02:00
parent 01569b6a0d
commit 16fc22387f
9 changed files with 312 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -7,14 +9,12 @@ import lombok.NoArgsConstructor;
import java.util.Map;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DossierStats {
private String dossierId;
private int numberOfFiles;
private int numberOfSoftDeletedFiles;
@ -25,6 +25,6 @@ public class DossierStats {
private boolean hasSuggestionsFilePresent; // true if at least one file in the dossier has suggestions
private boolean hasUpdatesFilePresent; //true if at least one file in the dossier has updates
private boolean hasNoFlagsFilePresent; // true if at least one file in the dossier has none of the other flags
private Map<ProcessingStatus,Integer> fileCountPerProcessingStatus;
private Map<WorkflowStatus,Integer> fileCountPerWorkflowStatus;
private Map<ProcessingStatus, Integer> fileCountPerProcessingStatus;
private Map<WorkflowStatus, Integer> fileCountPerWorkflowStatus;
}

29
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/metrics/HibernateStatisticsInterceptor.java Normal file
View File

@ -0,0 +1,29 @@
package com.iqser.red.service.peristence.v1.server.metrics;
import org.hibernate.EmptyInterceptor;
public class HibernateStatisticsInterceptor extends EmptyInterceptor {
private ThreadLocal<Integer> queryCount = new ThreadLocal<>();
public void startCounter() {
queryCount.set(0);
}
public Integer getQueryCount() {
return queryCount.get();
}
public void clearCounter() {
queryCount.remove();
}
@Override
public String onPrepareStatement(String sql) {
Integer count = queryCount.get();
if (count != null) {
queryCount.set(count + 1);
}
return super.onPrepareStatement(sql);
}
}

30
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/metrics/MetricsConfiguration.java Normal file
View File

@ -0,0 +1,30 @@
package com.iqser.red.service.peristence.v1.server.metrics;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.orm.jpa.HibernatePropertiesCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import java.util.Map;
@Configuration
@RequiredArgsConstructor
@ComponentScan(basePackageClasses = MetricsConfiguration.class)
@ConditionalOnProperty(value = "metrics.persistence.enabled", havingValue = "true")
public class MetricsConfiguration implements HibernatePropertiesCustomizer {
@Override
public void customize(Map<String, Object> hibernateProperties) {
hibernateProperties.put("hibernate.session_factory.interceptor", hibernateInterceptor());
}
@Bean
public HibernateStatisticsInterceptor hibernateInterceptor() {
return new HibernateStatisticsInterceptor();
}
}

86
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/metrics/PersistenceMetricsAspect.java Normal file
View File

@ -0,0 +1,86 @@
package com.iqser.red.service.peristence.v1.server.metrics;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.concurrent.TimeUnit;
@Slf4j
@Aspect
@Component
@RequiredArgsConstructor
@ConditionalOnProperty(value = "metrics.persistence.enabled", havingValue = "true")
public class PersistenceMetricsAspect {
private final MeterRegistry meterRegistry;
private final HibernateStatisticsInterceptor statisticsInterceptor;
@PostConstruct
protected void postConstruct() {
log.warn("Persistence Metrics are enabled!");
}
@Pointcut("execution(public * org.springframework.data.repository.Repository+.*(..))")
public void monitor() {
}
@Around("monitor()")
public Object profile(ProceedingJoinPoint pjp) throws Throwable {
long start = System.currentTimeMillis();
try {
statisticsInterceptor.startCounter();
return pjp.proceed();
} finally {
Integer queryCount = statisticsInterceptor.getQueryCount();
statisticsInterceptor.clearCounter();
long elapsedTime = System.currentTimeMillis() - start;
try {
String repository = pjp.getTarget().getClass().getSimpleName();
if (pjp.getThis().getClass().getGenericInterfaces().length > 0) {
repository = pjp.getThis().getClass().getGenericInterfaces()[0].getTypeName();
}
String label = repository + ":" + pjp.getSignature().getName();
processQueryCounterSummary(label, queryCount);
processTimer(label, elapsedTime);
} catch (Exception e) {
log.debug("Processing Metrics failed", e);
}
}
}
private void processQueryCounterSummary(String label, int queryCount) {
final String metric = "QueryCounter:" + label;
final DistributionSummary summary = meterRegistry.find(metric).summary();
if (summary != null) {
summary.record(queryCount);
} else {
meterRegistry.summary(metric, "JPA", "QueryCount").record(queryCount);
}
}
private void processTimer(String label, long elapsedTime) {
final String metric = "Timer:" + label;
final Timer foundCounter = meterRegistry.find(metric).timer();
if (foundCounter != null) {
foundCounter.record(elapsedTime, TimeUnit.MILLISECONDS);
} else {
meterRegistry.timer(metric, "JPA", "Timer").record(elapsedTime, TimeUnit.MILLISECONDS);
}
}
}

2
persistence-service-v1/persistence-service-server-v1/src/main/resources/application-dev.yml
View File

@ -25,3 +25,5 @@ spring:
hibernate:
ddl-auto: none
naming-strategy: org.hibernate.cfg.ImprovedNamingStrategy
monitoring:enabled: true

5
persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml
View File

@ -48,9 +48,12 @@ management:
metrics.enabled: ${monitoring.enabled:false}
prometheus.enabled: ${monitoring.enabled:false}
health.enabled: true
endpoints.web.exposure.include: prometheus, health
endpoints.web.exposure.include: prometheus, health, metrics
metrics.export.prometheus.enabled: ${monitoring.enabled:false}
metrics:
persistence:
enabled: ${monitoring.enabled:false}
storage:
signer-type: 'AWSS3V4SignerType'

9
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -5,6 +5,7 @@ import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.client.SearchClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
@ -114,6 +115,8 @@ public abstract class AbstractPersistenceServerServiceTest {
private NotificationPreferencesRepository notificationPreferencesRepository;
@Autowired
private DossierStatusRepository dossierStatusRepository;
@Autowired
private MetricsPrinterService metricsPrinterService;
@Before
public void setupOptimize() {
@ -151,6 +154,12 @@ public abstract class AbstractPersistenceServerServiceTest {
((FileSystemBackedStorageService) this.storageService).clearStorage();
}
@After
public void printMetrics() {
this.metricsPrinterService.printMetrics();
}
@After
public void afterTests() {
dossierAttributeRepository.deleteAll();

133
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/MetricsPrinterService.java Normal file
View File

@ -0,0 +1,133 @@
package com.iqser.red.service.peristence.v1.server.utils;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
@Service
public class MetricsPrinterService {
@Value("${server.port}")
private int serverPort;
@SneakyThrows
public void printMetrics() {
RestTemplate rt = new RestTemplate();
var url = "http://127.0.0.1:" + serverPort + "/actuator/metrics";
var ent = rt.getForEntity(url, JsonNode.class);
List<String> metrics = new ArrayList<>();
ent.getBody().get("names").forEach(name -> {
if (name.asText().startsWith("Timer") || name.asText().startsWith("QueryCounter")) {
metrics.add(name.asText());
}
});
System.out.println("Metrics: "+metrics);
double count = 0;
double total = 0;
var counterResults = new ArrayList<Result>();
var timerResults = new ArrayList<Result>();
for (var metric : metrics) {
if (!metric.contains("com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.")) {
continue;
}
if (metric.startsWith("Timer")) {
var m = rt.getForEntity(url +"/"+ metric, JsonNode.class);
var baseUnit = m.getBody().get("baseUnit").asText();
// System.out.println(om.writeValueAsString(m.getBody()));
for (var mes : m.getBody().get("measurements")) {
if (mes.get("statistic").asText().equals("COUNT")) {
count = mes.get("value").asDouble();
}
if (mes.get("statistic").asText().equals("TOTAL_TIME")) {
total = mes.get("value").asDouble() * 1000; // in ms
}
}
var result = new Result(metric, count, total);
timerResults.add(result);
}
if (metric.startsWith("Query")) {
var m = rt.getForEntity(url +"/"+ metric, JsonNode.class);
for (var mes : m.getBody().get("measurements")) {
if (mes.get("statistic").asText().equals("COUNT")) {
count = mes.get("value").asDouble();
}
if (mes.get("statistic").asText().equals("TOTAL")) {
total = mes.get("value").asDouble(); // in ms
}
}
var result = new Result(metric, count, total);
counterResults.add(result);
}
}
timerResults.sort((a, b) -> Double.compare(b.total, a.total));
timerResults.forEach(System.out::println);
counterResults.sort((a, b) -> Double.compare(b.average, a.average));
counterResults.forEach(System.out::println);
}
@Slf4j
public static class Result {
String metric;
double count;
double total;
double average;
String totalStr;
String averageStr;
public Result(String metric, double count, double total) {
this.metric = metric.replace("com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.", "");
this.count = count;
this.total = total;
this.average = total / count;
DecimalFormat df = new DecimalFormat("#.##");
this.totalStr = df.format(this.total);
this.averageStr = df.format(this.average);
}
private String padRight(String str) {
while (str.length() < 100) {
str += " ";
}
return str;
}
private String padLeft(String str) {
while (str.length() < 12) {
str = " " + str;
}
return str;
}
@Override
public String toString() {
return padRight(metric + ": ") + " | " + padLeft(count + "") + " | " + padLeft(totalStr + "") + " | " + padLeft(averageStr + "");
}
}
}

15
persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml
View File

@ -1,3 +1,5 @@
monitoring.enabled: true
spring:
datasource:
@ -47,9 +49,20 @@ storage:
secret: minioadmin
backend: 's3'
server:
port: 28080
persistence-service:
imageServiceEnabled: false
metrics:
persistence:
enabled: true
management:
endpoint:
metrics.enabled: true
health.enabled: true
endpoints.web.exposure.include: prometheus, health, metrics