Merge branch 'RED-9349-rule-log' into 'master'

RED-9349 - Add possibility to log stuff in the rules

Closes RED-9349

See merge request redactmanager/redaction-service!446
This commit is contained in:
Andrei Isvoran 2024-06-26 10:17:58 +02:00
commit 990c09cfad
37 changed files with 631 additions and 18 deletions

View File

@ -20,6 +20,7 @@ val persistenceServiceVersion = "2.444.0"
val springBootStarterVersion = "3.1.5" val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4" val springCloudVersion = "4.0.4"
val testContainersVersion = "1.19.7" val testContainersVersion = "1.19.7"
val tomcatVersion = "10.1.18"
configurations { configurations {
all { all {
@ -41,6 +42,7 @@ dependencies {
implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
implementation("com.iqser.red.commons:storage-commons:2.45.0") implementation("com.iqser.red.commons:storage-commons:2.45.0")
implementation("com.knecon.fforesight:keycloak-commons:0.29.0")
implementation("com.knecon.fforesight:tenant-commons:0.24.0") implementation("com.knecon.fforesight:tenant-commons:0.24.0")
implementation("com.knecon.fforesight:tracing-commons:0.5.0") implementation("com.knecon.fforesight:tracing-commons:0.5.0")
@ -60,6 +62,11 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}") implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}")
implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}") implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
implementation("org.springframework.boot:spring-boot-starter-websocket:${springBootStarterVersion}")
implementation("org.springframework.security:spring-security-messaging:6.1.3")
implementation("org.apache.tomcat:tomcat-websocket:${tomcatVersion}")
implementation("org.apache.tomcat.embed:tomcat-embed-core:${tomcatVersion}")
implementation("net.logstash.logback:logstash-logback-encoder:7.4") implementation("net.logstash.logback:logstash-logback-encoder:7.4")
implementation("ch.qos.logback:logback-classic") implementation("ch.qos.logback:logback-classic")

View File

@ -20,6 +20,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.keycloakcommons.DefaultKeyCloakCommonsAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration; import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase; import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration; import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@ -32,7 +33,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@EnableCaching @EnableCaching
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class}) @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class}) @Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class) @EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class) @EnableConfigurationProperties(RedactionServiceSettings.class)

View File

@ -0,0 +1,95 @@
package com.iqser.red.service.redaction.v1.server.config;
import java.util.Collections;
import java.util.Optional;
import org.apache.tomcat.websocket.server.WsSci;
import org.springframework.boot.web.embedded.tomcat.TomcatContextCustomizer;
import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.simp.config.ChannelRegistration;
import org.springframework.messaging.simp.config.MessageBrokerRegistry;
import org.springframework.messaging.simp.stomp.StompCommand;
import org.springframework.messaging.simp.stomp.StompHeaderAccessor;
import org.springframework.messaging.support.ChannelInterceptor;
import org.springframework.messaging.support.MessageHeaderAccessor;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.oauth2.server.resource.authentication.BearerTokenAuthenticationToken;
import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken;
import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Configuration
@EnableWebSocketMessageBroker
@RequiredArgsConstructor
public class WebSocketConfiguration implements WebSocketMessageBrokerConfigurer {
private final TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
@Override
public void configureMessageBroker(MessageBrokerRegistry config) {
config.enableSimpleBroker("/topic");
config.setApplicationDestinationPrefixes("/app");
}
@Override
public void registerStompEndpoints(StompEndpointRegistry registry) {
registry.addEndpoint("/api/rules-logging/rulesocket").setAllowedOrigins("*");
}
@Override
public void configureClientInboundChannel(ChannelRegistration registration) {
// https://docs.spring.io/spring-framework/reference/web/websocket/stomp/authentication-token-based.html
registration.interceptors(new ChannelInterceptor() {
@Override
public Message<?> preSend(Message<?> message, MessageChannel channel) {
StompHeaderAccessor accessor = MessageHeaderAccessor.getAccessor(message, StompHeaderAccessor.class);
if (StompCommand.CONNECT.equals(accessor.getCommand())) {
Optional.ofNullable(accessor.getNativeHeader("Authorization"))
.ifPresent(ah -> {
String bearerToken = ah.get(0).replace("Bearer ", "");
log.info("Received bearer token {}", bearerToken);
AuthenticationManager authenticationManager = tenantAuthenticationManagerResolver.resolve(bearerToken);
JwtAuthenticationToken token = (JwtAuthenticationToken) authenticationManager.authenticate(new BearerTokenAuthenticationToken(bearerToken));
accessor.setUser(token);
});
}
return message;
}
});
}
@Bean
public TomcatServletWebServerFactory tomcatContainerFactory() {
TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory();
factory.setTomcatContextCustomizers(Collections.singletonList(tomcatContextCustomizer()));
return factory;
}
@Bean
public TomcatContextCustomizer tomcatContextCustomizer() {
return context -> context.addServletContainerInitializer(new WsSci(), null);
}
}

View File

@ -0,0 +1,88 @@
package com.iqser.red.service.redaction.v1.server.config;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.Message;
import org.springframework.messaging.simp.SimpMessageType;
import org.springframework.messaging.simp.stomp.StompHeaderAccessor;
import org.springframework.security.config.annotation.web.messaging.MessageSecurityMetadataSourceRegistry;
import org.springframework.security.config.annotation.web.socket.AbstractSecurityWebSocketMessageBrokerConfigurer;
import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken;
import com.knecon.fforesight.keycloakcommons.security.TokenUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Configuration
public class WebSocketSecurityConfiguration extends AbstractSecurityWebSocketMessageBrokerConfigurer {
@Value("${cors.enabled:false}")
private boolean corsEnabled;
@Override
protected void configureInbound(MessageSecurityMetadataSourceRegistry messages) {
messages.simpTypeMatchers(SimpMessageType.HEARTBEAT, SimpMessageType.UNSUBSCRIBE, SimpMessageType.DISCONNECT)
.permitAll()
.simpTypeMatchers(SimpMessageType.CONNECT)
.anonymous() // this is intended, see WebSocketConfiguration.configureClientInboundChannel
.simpTypeMatchers(SimpMessageType.SUBSCRIBE)
.access("@tenantWebSocketSecurityMatcher.checkCanSubscribeTo(authentication,message)")
.anyMessage()
.denyAll();
}
@Override
protected boolean sameOriginDisabled() {
return corsEnabled;
}
@Bean
public TenantWebSocketSecurityMatcher tenantWebSocketSecurityMatcher() {
return new TenantWebSocketSecurityMatcher();
}
public class TenantWebSocketSecurityMatcher {
public boolean checkCanSubscribeTo(JwtAuthenticationToken authentication, Message<?> message) {
var targetedTenant = extractTenantId(message);
var currentTenant = getCurrentTenant(authentication);
return targetedTenant.isPresent() && currentTenant.isPresent() && currentTenant.get().equals(targetedTenant.get());
}
private Optional<String> getCurrentTenant(JwtAuthenticationToken authentication) {
if (authentication != null && authentication.getToken() != null && authentication.getToken().getTokenValue() != null) {
return Optional.of(TokenUtils.toTenant(authentication.getToken().getTokenValue()));
} else {
return Optional.empty();
}
}
}
private Optional<String> extractTenantId(Message<?> message) {
StompHeaderAccessor sha = StompHeaderAccessor.wrap(message);
String topic = sha.getDestination();
if (topic == null) {
return Optional.empty();
}
String tenant = topic.split("/")[2];
return Optional.of(tenant);
}
}

View File

@ -0,0 +1,23 @@
package com.iqser.red.service.redaction.v1.server.logger;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@AllArgsConstructor
@NoArgsConstructor
@Getter
@ToString
public final class Context {
private String fileId;
private String dossierId;
private String dossierTemplateId;
@Setter
private long ruleVersion;
private int analysisNumber;
private String tenantId;
}

View File

@ -0,0 +1,45 @@
package com.iqser.red.service.redaction.v1.server.logger;
import java.time.OffsetDateTime;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;
@NoArgsConstructor
@AllArgsConstructor
@Data
@Builder
@ToString
public class RuleLogEvent {
private String tenantId;
private String fileId;
private String dossierId;
private String dossierTemplateId;
private long ruleVersion;
private int analysisNumber;
private OffsetDateTime timeStamp;
private LogLevel logLevel;
private String message;
}
@Getter
enum LogLevel {
INFO("INFO"),
WARN("WARN"),
ERROR("ERROR");
private final String name;
LogLevel(String name) {
this.name = name;
}
}

View File

@ -0,0 +1,72 @@
package com.iqser.red.service.redaction.v1.server.logger;
import java.time.OffsetDateTime;
import java.util.regex.Pattern;
import com.iqser.red.service.redaction.v1.server.service.WebSocketService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class RulesLogger {
private final WebSocketService webSocketService;
public void info(Context context, String message, Object... args) {
log(LogLevel.INFO, context, message, args);
}
public void warn(Context context, String message, Object... args) {
log(LogLevel.WARN, context, message, args);
}
public void error(Context context, String message, Throwable throwable, Object... args) {
log(LogLevel.ERROR, context, message + " Exception: " + throwable.toString(), args);
}
private void log(LogLevel logLevel, Context context, String message, Object... args) {
var formattedMessage = formatMessage(message, args);
var ruleLog = RuleLogEvent.builder()
.tenantId(context.getTenantId())
.ruleVersion(context.getRuleVersion())
.fileId(context.getFileId())
.analysisNumber(context.getAnalysisNumber())
.dossierId(context.getDossierId())
.dossierTemplateId(context.getDossierTemplateId())
.message(formattedMessage)
.logLevel(logLevel)
.timeStamp(OffsetDateTime.now())
.build();
webSocketService.sendLogEvent(ruleLog);
}
private String formatMessage(String message, Object... args) {
if (args == null || args.length == 0) {
return message;
}
var pattern = Pattern.compile("\\{}");
var matcher = pattern.matcher(message);
var sb = new StringBuilder();
int i = 0;
while (matcher.find() && i < args.length) {
matcher.appendReplacement(sb, args[i] != null ? args[i].toString() : "null");
i++;
}
matcher.appendTail(sb);
return sb.toString();
}
}

View File

@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper; import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Component;
@ -46,6 +47,7 @@ import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExec
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed; import io.micrometer.observation.annotation.Observed;
@ -93,6 +95,8 @@ public class AnalyzeService {
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Context context = new Context(analyzeRequest.getFileId(), analyzeRequest.getDossierId(), analyzeRequest.getDossierTemplateId(), 0, analyzeRequest.getAnalysisNumber(), TenantContext.getTenantId());
// not yet ready for reanalysis // not yet ready for reanalysis
if (entityLogWithoutEntries == null || document == null || document.getNumberOfPages() == 0) { if (entityLogWithoutEntries == null || document == null || document.getNumberOfPages() == 0) {
return analyze(analyzeRequest); return analyze(analyzeRequest);
@ -128,12 +132,15 @@ public class AnalyzeService {
document, document,
document.getNumberOfPages(), document.getNumberOfPages(),
true, true,
Collections.emptySet()); Collections.emptySet(),
context);
} }
KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
context.setRuleVersion(kieWrapperEntityRules.rulesVersion());
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -158,7 +165,8 @@ public class AnalyzeService {
dictionary, dictionary,
analyzeRequest.getFileAttributes(), analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(), analyzeRequest.getManualRedactions(),
nerEntities); nerEntities,
context);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
@ -177,7 +185,8 @@ public class AnalyzeService {
document, document,
document.getNumberOfPages(), document.getNumberOfPages(),
true, true,
new HashSet<>(allFileAttributes)); new HashSet<>(allFileAttributes),
context);
} }
@ -193,6 +202,8 @@ public class AnalyzeService {
var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT); var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Context context = new Context(analyzeRequest.getFileId(), analyzeRequest.getDossierId(), analyzeRequest.getDossierTemplateId(), kieWrapperEntityRules.rulesVersion(), analyzeRequest.getAnalysisNumber(), TenantContext.getTenantId());
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -223,7 +234,8 @@ public class AnalyzeService {
dictionary, dictionary,
analyzeRequest.getFileAttributes(), analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(), analyzeRequest.getManualRedactions(),
nerEntities); nerEntities,
context);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
@ -241,7 +253,8 @@ public class AnalyzeService {
document, document,
document.getNumberOfPages(), document.getNumberOfPages(),
false, false,
new HashSet<>(allFileAttributes)); new HashSet<>(allFileAttributes),
context);
} }
@ -252,7 +265,8 @@ public class AnalyzeService {
Document document, Document document,
int numberOfPages, int numberOfPages,
boolean isReanalysis, boolean isReanalysis,
Set<FileAttribute> addedFileAttributes) { Set<FileAttribute> addedFileAttributes,
Context context) {
EntityLog entityLog = entityLogChanges.getEntityLog(); EntityLog entityLog = entityLogChanges.getEntityLog();
@ -274,7 +288,7 @@ public class AnalyzeService {
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog); computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
long duration = System.currentTimeMillis() - startTime; long duration = System.currentTimeMillis() - startTime;
@ -305,7 +319,8 @@ public class AnalyzeService {
KieWrapper kieWrapperComponentRules, KieWrapper kieWrapperComponentRules,
Document document, Document document,
Set<FileAttribute> addedFileAttributes, Set<FileAttribute> addedFileAttributes,
EntityLog entityLog) { EntityLog entityLog,
Context context) {
if (!kieWrapperComponentRules.isPresent()) { if (!kieWrapperComponentRules.isPresent()) {
return; return;
@ -318,7 +333,8 @@ public class AnalyzeService {
entityLog, entityLog,
document, document,
addedFileAttributes, addedFileAttributes,
analyzeRequest.getComponentMappings()); analyzeRequest.getComponentMappings(),
context);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

View File

@ -0,0 +1,24 @@
package com.iqser.red.service.redaction.v1.server.service;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.logger.RuleLogEvent;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class WebSocketService {
private final SimpMessagingTemplate template;
public void sendLogEvent(RuleLogEvent ruleLogEvent) {
String destination = "/topic/" + ruleLogEvent.getTenantId() + "/rule-log-events";
log.info("Sending message to url {}", destination);
template.convertAndSend(destination, ruleLogEvent);
}
}

View File

@ -22,9 +22,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.WebSocketService;
import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingMemoryCache; import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingMemoryCache;
import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService; import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService;
import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator; import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
@ -47,19 +50,24 @@ public class ComponentDroolsExecutionService {
RedactionServiceSettings settings; RedactionServiceSettings settings;
ComponentMappingMemoryCache componentMappingMemoryCache; ComponentMappingMemoryCache componentMappingMemoryCache;
WebSocketService webSocketService;
public List<Component> executeRules(KieContainer kieContainer, public List<Component> executeRules(KieContainer kieContainer,
EntityLog entityLog, EntityLog entityLog,
Document document, Document document,
Set<FileAttribute> fileAttributes, Set<FileAttribute> fileAttributes,
List<ComponentMappingMetadata> componentMappings) { List<ComponentMappingMetadata> componentMappings,
Context context) {
KieSession kieSession = kieContainer.newKieSession(); KieSession kieSession = kieContainer.newKieSession();
ComponentCreationService componentCreationService = new ComponentCreationService(kieSession); ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
ComponentMappingService componentMappingService = new ComponentMappingService(componentMappingMemoryCache, componentMappings); ComponentMappingService componentMappingService = new ComponentMappingService(componentMappingMemoryCache, componentMappings);
RulesLogger logger = new RulesLogger(webSocketService);
kieSession.setGlobal("componentCreationService", componentCreationService); kieSession.setGlobal("componentCreationService", componentCreationService);
kieSession.setGlobal("logger", logger);
kieSession.setGlobal("context", context);
if (hasComponentMappingServiceGlobal(kieSession)) { if (hasComponentMappingServiceGlobal(kieSession)) {
kieSession.setGlobal(COMPONENT_MAPPING_SERVICE_GLOBAL, componentMappingService); kieSession.setGlobal(COMPONENT_MAPPING_SERVICE_GLOBAL, componentMappingService);
@ -73,9 +81,7 @@ public class ComponentDroolsExecutionService {
entities.add(Entity.fromEntityLogEntry(entry, document, entry.getStartOffset(), entry.getEndOffset())); entities.add(Entity.fromEntityLogEntry(entry, document, entry.getStartOffset(), entry.getEndOffset()));
if (entry.getDuplicatedTextRanges() != null && !entry.getDuplicatedTextRanges().isEmpty()) { if (entry.getDuplicatedTextRanges() != null && !entry.getDuplicatedTextRanges().isEmpty()) {
entry.getDuplicatedTextRanges() entry.getDuplicatedTextRanges()
.forEach(duplicatedTextRange -> { .forEach(duplicatedTextRange -> entities.add(Entity.fromEntityLogEntry(entry, document, duplicatedTextRange.getStart(), duplicatedTextRange.getEnd())));
entities.add(Entity.fromEntityLogEntry(entry, document, duplicatedTextRange.getStart(), duplicatedTextRange.getEnd()));
});
} }
return entities.stream(); return entities.stream();
}) })
@ -97,12 +103,14 @@ public class ComponentDroolsExecutionService {
completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS) completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS)
.get(); .get();
} catch (ExecutionException e) { } catch (ExecutionException e) {
logger.error(context, "Exception during rule execution", e);
kieSession.dispose(); kieSession.dispose();
if (e.getCause() instanceof TimeoutException) { if (e.getCause() instanceof TimeoutException) {
throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT); throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
} }
throw new RuntimeException(e); throw new RuntimeException(e);
} catch (InterruptedException e) { } catch (InterruptedException e) {
logger.error(context, "Exception during rule execution", e);
kieSession.dispose(); kieSession.dispose();
throw new RuntimeException(e); throw new RuntimeException(e);
} }

View File

@ -21,11 +21,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.service.WebSocketService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
@ -48,6 +51,7 @@ public class EntityDroolsExecutionService {
ObservationRegistry observationRegistry; ObservationRegistry observationRegistry;
ManualChangesApplicationService manualChangesApplicationService; ManualChangesApplicationService manualChangesApplicationService;
RedactionServiceSettings settings; RedactionServiceSettings settings;
WebSocketService webSocketService;
@Timed("redactmanager_executeRules") @Timed("redactmanager_executeRules")
@ -57,7 +61,8 @@ public class EntityDroolsExecutionService {
Dictionary dictionary, Dictionary dictionary,
List<FileAttribute> fileAttributes, List<FileAttribute> fileAttributes,
ManualRedactions manualRedactions, ManualRedactions manualRedactions,
NerEntities nerEntities) { NerEntities nerEntities,
Context context) {
return executeRules(kieContainer, return executeRules(kieContainer,
document, document,
@ -66,7 +71,8 @@ public class EntityDroolsExecutionService {
dictionary, dictionary,
fileAttributes, fileAttributes,
manualRedactions, manualRedactions,
nerEntities); nerEntities,
context);
} }
@ -78,7 +84,8 @@ public class EntityDroolsExecutionService {
Dictionary dictionary, Dictionary dictionary,
List<FileAttribute> fileAttributes, List<FileAttribute> fileAttributes,
ManualRedactions manualRedactions, ManualRedactions manualRedactions,
NerEntities nerEntities) { NerEntities nerEntities,
Context context) {
addNumberOfPagesAndSectionsToAnalyseToTrace(document.getNumberOfPages(), sectionsToAnalyze.size()); addNumberOfPagesAndSectionsToAnalyseToTrace(document.getNumberOfPages(), sectionsToAnalyze.size());
@ -88,11 +95,14 @@ public class EntityDroolsExecutionService {
.count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document); .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession, nodesInKieSession); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession, nodesInKieSession);
RulesLogger logger = new RulesLogger(webSocketService);
kieSession.setGlobal("document", document); kieSession.setGlobal("document", document);
kieSession.setGlobal("entityCreationService", entityCreationService); kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
kieSession.setGlobal("dictionary", dictionary); kieSession.setGlobal("dictionary", dictionary);
kieSession.setGlobal("logger", logger);
kieSession.setGlobal("context", context);
kieSession.insert(document); kieSession.insert(document);
@ -132,12 +142,14 @@ public class EntityDroolsExecutionService {
completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS) completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS)
.get(); .get();
} catch (ExecutionException e) { } catch (ExecutionException e) {
logger.error(context, "Exception during rule execution", e);
kieSession.dispose(); kieSession.dispose();
if (e.getCause() instanceof TimeoutException) { if (e.getCause() instanceof TimeoutException) {
throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY); throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
} }
throw new RuntimeException(e); throw new RuntimeException(e);
} catch (InterruptedException e) { } catch (InterruptedException e) {
logger.error(context, "Exception during rule execution", e);
kieSession.dispose(); kieSession.dispose();
throw new RuntimeException(e); throw new RuntimeException(e);
} }

View File

@ -16,6 +16,14 @@ project.version: 1.0-SNAPSHOT
server: server:
port: 8080 port: 8080
fforesight:
keycloak:
enabled: true
ignored-endpoints: [ '/redaction-gateway-v1', '/actuator/health/**',"/api/rules-logging/rulesocket","/api/rules-logging/rulesocket/**", '/redaction-gateway-v1/async/download/with-ott/**',
'/internal-api/**', '/redaction-gateway-v1/docs/swagger-ui',
'/redaction-gateway-v1/docs/**','/redaction-gateway-v1/docs',
'/api', '/api/','/api/docs/**','/api/docs','/api/docs/swagger-ui' ]
spring: spring:
application: application:
name: redaction-service name: redaction-service

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@ -52,6 +54,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,8 +12,12 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService;
import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
@ -26,6 +30,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
global ComponentCreationService componentCreationService global ComponentCreationService componentCreationService
global ComponentMappingService componentMappingService
global RulesLogger logger
global Context context
/** /**
The imports, globals, queries and rules from this file are required for any component rule file. The imports, globals, queries and rules from this file are required for any component rule file.

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -59,6 +61,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
/** /**
The imports, globals, queries and rules from this file are required for any entity rule file. The imports, globals, queries and rules from this file are required for any entity rule file.

View File

@ -59,6 +59,7 @@ import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor; import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor;
import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider; import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
@ -191,6 +192,9 @@ public abstract class AbstractRedactionIntegrationTest {
@MockBean @MockBean
protected DictionaryClient dictionaryClient; protected DictionaryClient dictionaryClient;
@MockBean
protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
@BeforeEach @BeforeEach
public void setup() { public void setup() {

View File

@ -15,6 +15,7 @@ import java.util.stream.Stream;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieContainer;
import org.mockito.Mock;
import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@ -38,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers
import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient; import com.knecon.fforesight.tenantcommons.TenantsClient;
@ -55,6 +57,9 @@ public class DictionaryServiceTest {
@MockBean @MockBean
protected KieContainer kieContainer; protected KieContainer kieContainer;
@MockBean
protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
@MockBean @MockBean
protected DictionaryClient dictionaryClient; protected DictionaryClient dictionaryClient;

View File

@ -87,6 +87,7 @@ import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilitie
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@ -253,6 +254,8 @@ public class RulesTest {
private LegalBasisClient legalBasisClient; private LegalBasisClient legalBasisClient;
@MockBean @MockBean
private TenantsClient tenantsClient; private TenantsClient tenantsClient;
@MockBean
private TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
@BeforeEach @BeforeEach

View File

@ -34,6 +34,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
@ -154,12 +155,14 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart); System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart);
long droolsStart = System.currentTimeMillis(); long droolsStart = System.currentTimeMillis();
Context context = new Context("fileId", "dossierId", "dossierTemplateId", 1, 1, "redaction");
List<FileAttribute> fileAttributes = entityDroolsExecutionService.executeRules(kieContainer, List<FileAttribute> fileAttributes = entityDroolsExecutionService.executeRules(kieContainer,
document, document,
dictionary, dictionary,
Collections.emptyList(), Collections.emptyList(),
new ManualRedactions(), new ManualRedactions(),
new NerEntities()); new NerEntities(),
context);
System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart); System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart);
System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart); System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart);

View File

@ -0,0 +1,85 @@
package com.iqser.red.service.redaction.v1.server.logger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import org.drools.io.ClassPathResource;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.ReleaseId;
import org.kie.api.runtime.KieContainer;
import org.kie.api.runtime.KieSession;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import com.iqser.red.service.redaction.v1.server.service.WebSocketService;
@ExtendWith(MockitoExtension.class)
class RulesLoggerTest {
private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
private final PrintStream originalOut = System.out;
@Mock
private WebSocketService webSocketService;
@InjectMocks
private RulesLogger logger;
@Test
void testRulesLogging() {
System.setOut(new PrintStream(outContent));
RulesLogger logger = new RulesLogger(webSocketService);
KieServices ks = KieServices.Factory.get();
KieFileSystem kfs = ks.newKieFileSystem();
kfs.write(new ClassPathResource("logs/rules_logging.drl"));
KieBuilder kieBuilder = ks.newKieBuilder(kfs).buildAll();
ReleaseId releaseId = kieBuilder.getKieModule().getReleaseId();
KieContainer kContainer = ks.newKieContainer(releaseId);
KieSession kSession = kContainer.newKieSession();
Context context = new Context("fileId", "dossierId", "dossierTemplateId", 1, 1, "redaction");
kSession.setGlobal("logger", logger);
kSession.setGlobal("context", context);
try {
kSession.fireAllRules();
} catch (Exception e) {
logger.error(context, "Exception during rule execution", e);
} finally {
kSession.dispose();
}
System.setOut(originalOut);
System.out.println(outContent);
ArgumentCaptor<RuleLogEvent> argument = ArgumentCaptor.forClass(RuleLogEvent.class);
verify(webSocketService, times(3)).sendLogEvent(argument.capture());
assertEquals(argument.getAllValues().get(0).getLogLevel(), LogLevel.INFO);
assertEquals(argument.getAllValues().get(0).getRuleVersion(), 1);
assertEquals(argument.getAllValues().get(0).getTenantId(), "redaction");
assertEquals(argument.getAllValues().get(0).getFileId(), "fileId");
assertEquals(argument.getAllValues().get(0).getDossierId(), "dossierId");
assertEquals(argument.getAllValues().get(0).getAnalysisNumber(), 1);
assertEquals(argument.getAllValues().get(0).getDossierTemplateId(), "dossierTemplateId");
assertEquals(argument.getAllValues().get(0).getMessage(), "This is a test log placeholder");
assertEquals(argument.getAllValues().get(1).getLogLevel(), LogLevel.WARN);
assertEquals(argument.getAllValues().get(1).getMessage(), "This is a warning log with multiple placeholders p1 p2 p3");
assertEquals(argument.getAllValues().get(2).getLogLevel(), LogLevel.ERROR);
assertEquals(argument.getAllValues().get(2).getMessage(), "Exception during rule execution Exception: Exception executing consequence for rule \"LOG.0.2: Test log error\" in drools: java.lang.NullPointerException: Cannot invoke \"String.toString()\" because \"result\" is null");
}
}

View File

@ -56,6 +56,7 @@ import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient; import com.knecon.fforesight.tenantcommons.TenantsClient;
@ -91,6 +92,9 @@ public class LiveDataIntegrationTest {
@MockBean @MockBean
private LegalBasisClient legalBasisClient; private LegalBasisClient legalBasisClient;
@MockBean
protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
@Autowired @Autowired
private ResourcePatternResolver resourcePatternResolver; private ResourcePatternResolver resourcePatternResolver;

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@ -52,6 +54,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,9 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService; import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService;
@ -28,6 +31,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
global ComponentCreationService componentCreationService global ComponentCreationService componentCreationService
global ComponentMappingService componentMappingService global ComponentMappingService componentMappingService
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,9 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
@ -26,6 +29,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
global ComponentCreationService componentCreationService global ComponentCreationService componentCreationService
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,9 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
@ -26,6 +29,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
global ComponentCreationService componentCreationService global ComponentCreationService componentCreationService
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -0,0 +1,32 @@
package drools
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
global RulesLogger logger
global Context context
rule "LOG.0.0: Test log info"
salience 1
when
eval(true)
then
logger.info(context, "This is a test log {}", "placeholder");
end
rule "LOG.0.1: Test log warn"
salience 1
when
eval(true)
then
logger.warn(context, "This is a warning log with multiple placeholders {} {} {}", "p1", "p2", "p3");
end
rule "LOG.0.2: Test log error"
when
eval(true)
then
String result = null;
result.toString();
end

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.document.graph.*; import com.iqser.red.service.redaction.v1.server.document.graph.*;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section;
@ -55,6 +57,8 @@ global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global NerEntitiesAdapter nerEntitiesAdapter global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
@ -60,6 +62,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------

View File

@ -12,6 +12,8 @@ import java.util.Collection;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.Optional; import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@ -52,6 +54,8 @@ global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger
global Context context
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------