From f3ae68a8dac474d85d58e3aaafeb8de33d575ea4 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Wed, 26 Jun 2024 10:17:58 +0200 Subject: [PATCH] RED-9349 - Add possibility to log stuff in the rules --- .../build.gradle.kts | 7 ++ .../redaction/v1/server/Application.java | 3 +- .../server/config/WebSocketConfiguration.java | 95 +++++++++++++++++++ .../WebSocketSecurityConfiguration.java | 88 +++++++++++++++++ .../redaction/v1/server/logger/Context.java | 23 +++++ .../v1/server/logger/RuleLogEvent.java | 45 +++++++++ .../v1/server/logger/RulesLogger.java | 72 ++++++++++++++ .../v1/server/service/AnalyzeService.java | 34 +++++-- .../v1/server/service/WebSocketService.java | 24 +++++ .../ComponentDroolsExecutionService.java | 16 +++- .../drools/EntityDroolsExecutionService.java | 18 +++- .../src/main/resources/application.yml | 8 ++ .../resources/drools/all_rules_documine.drl | 4 + .../resources/drools/base_component_rules.drl | 7 ++ .../src/main/resources/drools/base_rules.drl | 4 + .../AbstractRedactionIntegrationTest.java | 4 + .../v1/server/DictionaryServiceTest.java | 5 + .../redaction/v1/server/RulesTest.java | 3 + .../DocumentPerformanceIntegrationTest.java | 5 +- .../v1/server/logger/RulesLoggerTest.java | 85 +++++++++++++++++ .../realdata/LiveDataIntegrationTest.java | 4 + .../resources/drools/acceptance_rules.drl | 4 + .../drools/all_redact_manager_rules.drl | 4 + .../test/resources/drools/documine_flora.drl | 4 + .../drools/documine_flora_components.drl | 5 + .../resources/drools/efsa_sanitisation.drl | 4 + .../drools/manual_redaction_rules.drl | 4 + .../src/test/resources/drools/rules.drl | 4 + .../src/test/resources/drools/rules_v2.drl | 4 + .../src/test/resources/drools/table_demo.drl | 4 + .../drools/table_demo_components.drl | 5 + .../test/resources/drools/test_components.drl | 5 + .../src/test/resources/drools/test_rules.drl | 4 + .../src/test/resources/logs/rules_logging.drl | 32 +++++++ .../EFSA_sanitisation_GFL_v1/rules.drl | 4 + .../resources/all_redact_manager_rules.drl | 4 + .../src/main/resources/all_rules_documine.drl | 4 + 37 files changed, 631 insertions(+), 18 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/config/WebSocketConfiguration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/config/WebSocketSecurityConfiguration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/Context.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/RuleLogEvent.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/RulesLogger.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/WebSocketService.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/logger/RulesLoggerTest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/logs/rules_logging.drl diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 9a39bdf1..342414bd 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -20,6 +20,7 @@ val persistenceServiceVersion = "2.444.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" +val tomcatVersion = "10.1.18" configurations { all { @@ -41,6 +42,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.45.0") + implementation("com.knecon.fforesight:keycloak-commons:0.29.0") implementation("com.knecon.fforesight:tenant-commons:0.24.0") implementation("com.knecon.fforesight:tracing-commons:0.5.0") @@ -60,6 +62,11 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}") implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}") + implementation("org.springframework.boot:spring-boot-starter-websocket:${springBootStarterVersion}") + implementation("org.springframework.security:spring-security-messaging:6.1.3") + implementation("org.apache.tomcat:tomcat-websocket:${tomcatVersion}") + implementation("org.apache.tomcat.embed:tomcat-embed-core:${tomcatVersion}") + implementation("net.logstash.logback:logstash-logback-encoder:7.4") implementation("ch.qos.logback:logback-classic") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java index bc16b1b6..eeddf593 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java @@ -20,6 +20,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.knecon.fforesight.keycloakcommons.DefaultKeyCloakCommonsAutoConfiguration; import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration; import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase; import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration; @@ -32,7 +33,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @EnableCaching -@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class}) +@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class}) @Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class}) @EnableFeignClients(basePackageClasses = RulesClient.class) @EnableConfigurationProperties(RedactionServiceSettings.class) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/config/WebSocketConfiguration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/config/WebSocketConfiguration.java new file mode 100644 index 00000000..55b9eaeb --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/config/WebSocketConfiguration.java @@ -0,0 +1,95 @@ +package com.iqser.red.service.redaction.v1.server.config; + +import java.util.Collections; +import java.util.Optional; + +import org.apache.tomcat.websocket.server.WsSci; +import org.springframework.boot.web.embedded.tomcat.TomcatContextCustomizer; +import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.simp.config.ChannelRegistration; +import org.springframework.messaging.simp.config.MessageBrokerRegistry; +import org.springframework.messaging.simp.stomp.StompCommand; +import org.springframework.messaging.simp.stomp.StompHeaderAccessor; +import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.messaging.support.MessageHeaderAccessor; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.oauth2.server.resource.authentication.BearerTokenAuthenticationToken; +import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken; +import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker; +import org.springframework.web.socket.config.annotation.StompEndpointRegistry; +import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer; + +import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Configuration +@EnableWebSocketMessageBroker +@RequiredArgsConstructor +public class WebSocketConfiguration implements WebSocketMessageBrokerConfigurer { + + private final TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; + + + @Override + public void configureMessageBroker(MessageBrokerRegistry config) { + + config.enableSimpleBroker("/topic"); + config.setApplicationDestinationPrefixes("/app"); + } + + + @Override + public void registerStompEndpoints(StompEndpointRegistry registry) { + + registry.addEndpoint("/api/rules-logging/rulesocket").setAllowedOrigins("*"); + } + + + @Override + public void configureClientInboundChannel(ChannelRegistration registration) { + + // https://docs.spring.io/spring-framework/reference/web/websocket/stomp/authentication-token-based.html + registration.interceptors(new ChannelInterceptor() { + @Override + public Message preSend(Message message, MessageChannel channel) { + + StompHeaderAccessor accessor = MessageHeaderAccessor.getAccessor(message, StompHeaderAccessor.class); + if (StompCommand.CONNECT.equals(accessor.getCommand())) { + Optional.ofNullable(accessor.getNativeHeader("Authorization")) + .ifPresent(ah -> { + String bearerToken = ah.get(0).replace("Bearer ", ""); + log.info("Received bearer token {}", bearerToken); + AuthenticationManager authenticationManager = tenantAuthenticationManagerResolver.resolve(bearerToken); + JwtAuthenticationToken token = (JwtAuthenticationToken) authenticationManager.authenticate(new BearerTokenAuthenticationToken(bearerToken)); + accessor.setUser(token); + }); + } + return message; + } + }); + } + + + @Bean + public TomcatServletWebServerFactory tomcatContainerFactory() { + + TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory(); + factory.setTomcatContextCustomizers(Collections.singletonList(tomcatContextCustomizer())); + return factory; + } + + + @Bean + public TomcatContextCustomizer tomcatContextCustomizer() { + + return context -> context.addServletContainerInitializer(new WsSci(), null); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/config/WebSocketSecurityConfiguration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/config/WebSocketSecurityConfiguration.java new file mode 100644 index 00000000..27f0d46f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/config/WebSocketSecurityConfiguration.java @@ -0,0 +1,88 @@ +package com.iqser.red.service.redaction.v1.server.config; + +import java.util.Optional; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.messaging.Message; +import org.springframework.messaging.simp.SimpMessageType; +import org.springframework.messaging.simp.stomp.StompHeaderAccessor; +import org.springframework.security.config.annotation.web.messaging.MessageSecurityMetadataSourceRegistry; +import org.springframework.security.config.annotation.web.socket.AbstractSecurityWebSocketMessageBrokerConfigurer; +import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken; + +import com.knecon.fforesight.keycloakcommons.security.TokenUtils; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Configuration +public class WebSocketSecurityConfiguration extends AbstractSecurityWebSocketMessageBrokerConfigurer { + + @Value("${cors.enabled:false}") + private boolean corsEnabled; + + + @Override + protected void configureInbound(MessageSecurityMetadataSourceRegistry messages) { + + messages.simpTypeMatchers(SimpMessageType.HEARTBEAT, SimpMessageType.UNSUBSCRIBE, SimpMessageType.DISCONNECT) + .permitAll() + .simpTypeMatchers(SimpMessageType.CONNECT) + .anonymous() // this is intended, see WebSocketConfiguration.configureClientInboundChannel + .simpTypeMatchers(SimpMessageType.SUBSCRIBE) + .access("@tenantWebSocketSecurityMatcher.checkCanSubscribeTo(authentication,message)") + .anyMessage() + .denyAll(); + } + + + @Override + protected boolean sameOriginDisabled() { + + return corsEnabled; + } + + + @Bean + public TenantWebSocketSecurityMatcher tenantWebSocketSecurityMatcher() { + + return new TenantWebSocketSecurityMatcher(); + } + + + public class TenantWebSocketSecurityMatcher { + + public boolean checkCanSubscribeTo(JwtAuthenticationToken authentication, Message message) { + + var targetedTenant = extractTenantId(message); + var currentTenant = getCurrentTenant(authentication); + return targetedTenant.isPresent() && currentTenant.isPresent() && currentTenant.get().equals(targetedTenant.get()); + } + + + private Optional getCurrentTenant(JwtAuthenticationToken authentication) { + + if (authentication != null && authentication.getToken() != null && authentication.getToken().getTokenValue() != null) { + return Optional.of(TokenUtils.toTenant(authentication.getToken().getTokenValue())); + } else { + return Optional.empty(); + } + } + + } + + private Optional extractTenantId(Message message) { + + StompHeaderAccessor sha = StompHeaderAccessor.wrap(message); + String topic = sha.getDestination(); + if (topic == null) { + return Optional.empty(); + } + + String tenant = topic.split("/")[2]; + return Optional.of(tenant); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/Context.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/Context.java new file mode 100644 index 00000000..c51e1b2c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/Context.java @@ -0,0 +1,23 @@ +package com.iqser.red.service.redaction.v1.server.logger; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +@ToString +public final class Context { + + private String fileId; + private String dossierId; + private String dossierTemplateId; + @Setter + private long ruleVersion; + private int analysisNumber; + private String tenantId; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/RuleLogEvent.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/RuleLogEvent.java new file mode 100644 index 00000000..5b269dda --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/RuleLogEvent.java @@ -0,0 +1,45 @@ +package com.iqser.red.service.redaction.v1.server.logger; + +import java.time.OffsetDateTime; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; + +@NoArgsConstructor +@AllArgsConstructor +@Data +@Builder +@ToString +public class RuleLogEvent { + + private String tenantId; + private String fileId; + private String dossierId; + private String dossierTemplateId; + private long ruleVersion; + private int analysisNumber; + private OffsetDateTime timeStamp; + private LogLevel logLevel; + private String message; + +} + +@Getter +enum LogLevel { + INFO("INFO"), + WARN("WARN"), + ERROR("ERROR"); + + private final String name; + + + LogLevel(String name) { + + this.name = name; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/RulesLogger.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/RulesLogger.java new file mode 100644 index 00000000..aae52fca --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/logger/RulesLogger.java @@ -0,0 +1,72 @@ +package com.iqser.red.service.redaction.v1.server.logger; + +import java.time.OffsetDateTime; +import java.util.regex.Pattern; + +import com.iqser.red.service.redaction.v1.server.service.WebSocketService; + +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class RulesLogger { + + private final WebSocketService webSocketService; + + public void info(Context context, String message, Object... args) { + + log(LogLevel.INFO, context, message, args); + } + + + public void warn(Context context, String message, Object... args) { + + log(LogLevel.WARN, context, message, args); + } + + + public void error(Context context, String message, Throwable throwable, Object... args) { + + log(LogLevel.ERROR, context, message + " Exception: " + throwable.toString(), args); + } + + + private void log(LogLevel logLevel, Context context, String message, Object... args) { + + var formattedMessage = formatMessage(message, args); + var ruleLog = RuleLogEvent.builder() + .tenantId(context.getTenantId()) + .ruleVersion(context.getRuleVersion()) + .fileId(context.getFileId()) + .analysisNumber(context.getAnalysisNumber()) + .dossierId(context.getDossierId()) + .dossierTemplateId(context.getDossierTemplateId()) + .message(formattedMessage) + .logLevel(logLevel) + .timeStamp(OffsetDateTime.now()) + .build(); + + webSocketService.sendLogEvent(ruleLog); + } + + + private String formatMessage(String message, Object... args) { + + if (args == null || args.length == 0) { + return message; + } + + var pattern = Pattern.compile("\\{}"); + var matcher = pattern.matcher(message); + var sb = new StringBuilder(); + int i = 0; + + while (matcher.find() && i < args.length) { + matcher.appendReplacement(sb, args[i] != null ? args[i].toString() : "null"); + i++; + } + matcher.appendTail(sb); + + return sb.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 5c5af376..0f5057c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.component.Component; @@ -46,6 +47,7 @@ import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExec import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; @@ -93,6 +95,8 @@ public class AnalyzeService { ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + Context context = new Context(analyzeRequest.getFileId(), analyzeRequest.getDossierId(), analyzeRequest.getDossierTemplateId(), 0, analyzeRequest.getAnalysisNumber(), TenantContext.getTenantId()); + // not yet ready for reanalysis if (entityLogWithoutEntries == null || document == null || document.getNumberOfPages() == 0) { return analyze(analyzeRequest); @@ -128,12 +132,15 @@ public class AnalyzeService { document, document.getNumberOfPages(), true, - Collections.emptySet()); + Collections.emptySet(), + context); } KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + context.setRuleVersion(kieWrapperEntityRules.rulesVersion()); + NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -158,7 +165,8 @@ public class AnalyzeService { dictionary, analyzeRequest.getFileAttributes(), analyzeRequest.getManualRedactions(), - nerEntities); + nerEntities, + context); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, @@ -177,7 +185,8 @@ public class AnalyzeService { document, document.getNumberOfPages(), true, - new HashSet<>(allFileAttributes)); + new HashSet<>(allFileAttributes), + context); } @@ -193,6 +202,8 @@ public class AnalyzeService { var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT); log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + Context context = new Context(analyzeRequest.getFileId(), analyzeRequest.getDossierId(), analyzeRequest.getDossierTemplateId(), kieWrapperEntityRules.rulesVersion(), analyzeRequest.getAnalysisNumber(), TenantContext.getTenantId()); + Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -223,7 +234,8 @@ public class AnalyzeService { dictionary, analyzeRequest.getFileAttributes(), analyzeRequest.getManualRedactions(), - nerEntities); + nerEntities, + context); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, @@ -241,7 +253,8 @@ public class AnalyzeService { document, document.getNumberOfPages(), false, - new HashSet<>(allFileAttributes)); + new HashSet<>(allFileAttributes), + context); } @@ -252,7 +265,8 @@ public class AnalyzeService { Document document, int numberOfPages, boolean isReanalysis, - Set addedFileAttributes) { + Set addedFileAttributes, + Context context) { EntityLog entityLog = entityLogChanges.getEntityLog(); @@ -274,7 +288,7 @@ public class AnalyzeService { log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog); + computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context); long duration = System.currentTimeMillis() - startTime; @@ -305,7 +319,8 @@ public class AnalyzeService { KieWrapper kieWrapperComponentRules, Document document, Set addedFileAttributes, - EntityLog entityLog) { + EntityLog entityLog, + Context context) { if (!kieWrapperComponentRules.isPresent()) { return; @@ -318,7 +333,8 @@ public class AnalyzeService { entityLog, document, addedFileAttributes, - analyzeRequest.getComponentMappings()); + analyzeRequest.getComponentMappings(), + context); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/WebSocketService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/WebSocketService.java new file mode 100644 index 00000000..a14295fa --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/WebSocketService.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import org.springframework.messaging.simp.SimpMessagingTemplate; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.server.logger.RuleLogEvent; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class WebSocketService { + + private final SimpMessagingTemplate template; + + public void sendLogEvent(RuleLogEvent ruleLogEvent) { + + String destination = "/topic/" + ruleLogEvent.getTenantId() + "/rule-log-events"; + log.info("Sending message to url {}", destination); + template.convertAndSend(destination, ruleLogEvent); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java index e145ddef..dbed5a5e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java @@ -22,9 +22,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.logger.Context; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.WebSocketService; import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingMemoryCache; import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService; import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator; @@ -47,19 +50,24 @@ public class ComponentDroolsExecutionService { RedactionServiceSettings settings; ComponentMappingMemoryCache componentMappingMemoryCache; + WebSocketService webSocketService; public List executeRules(KieContainer kieContainer, EntityLog entityLog, Document document, Set fileAttributes, - List componentMappings) { + List componentMappings, + Context context) { KieSession kieSession = kieContainer.newKieSession(); ComponentCreationService componentCreationService = new ComponentCreationService(kieSession); ComponentMappingService componentMappingService = new ComponentMappingService(componentMappingMemoryCache, componentMappings); + RulesLogger logger = new RulesLogger(webSocketService); kieSession.setGlobal("componentCreationService", componentCreationService); + kieSession.setGlobal("logger", logger); + kieSession.setGlobal("context", context); if (hasComponentMappingServiceGlobal(kieSession)) { kieSession.setGlobal(COMPONENT_MAPPING_SERVICE_GLOBAL, componentMappingService); @@ -73,9 +81,7 @@ public class ComponentDroolsExecutionService { entities.add(Entity.fromEntityLogEntry(entry, document, entry.getStartOffset(), entry.getEndOffset())); if (entry.getDuplicatedTextRanges() != null && !entry.getDuplicatedTextRanges().isEmpty()) { entry.getDuplicatedTextRanges() - .forEach(duplicatedTextRange -> { - entities.add(Entity.fromEntityLogEntry(entry, document, duplicatedTextRange.getStart(), duplicatedTextRange.getEnd())); - }); + .forEach(duplicatedTextRange -> entities.add(Entity.fromEntityLogEntry(entry, document, duplicatedTextRange.getStart(), duplicatedTextRange.getEnd()))); } return entities.stream(); }) @@ -97,12 +103,14 @@ public class ComponentDroolsExecutionService { completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS) .get(); } catch (ExecutionException e) { + logger.error(context, "Exception during rule execution", e); kieSession.dispose(); if (e.getCause() instanceof TimeoutException) { throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT); } throw new RuntimeException(e); } catch (InterruptedException e) { + logger.error(context, "Exception during rule execution", e); kieSession.dispose(); throw new RuntimeException(e); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index fbe4ccc9..7451057f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -21,11 +21,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.logger.Context; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.service.WebSocketService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; @@ -48,6 +51,7 @@ public class EntityDroolsExecutionService { ObservationRegistry observationRegistry; ManualChangesApplicationService manualChangesApplicationService; RedactionServiceSettings settings; + WebSocketService webSocketService; @Timed("redactmanager_executeRules") @@ -57,7 +61,8 @@ public class EntityDroolsExecutionService { Dictionary dictionary, List fileAttributes, ManualRedactions manualRedactions, - NerEntities nerEntities) { + NerEntities nerEntities, + Context context) { return executeRules(kieContainer, document, @@ -66,7 +71,8 @@ public class EntityDroolsExecutionService { dictionary, fileAttributes, manualRedactions, - nerEntities); + nerEntities, + context); } @@ -78,7 +84,8 @@ public class EntityDroolsExecutionService { Dictionary dictionary, List fileAttributes, ManualRedactions manualRedactions, - NerEntities nerEntities) { + NerEntities nerEntities, + Context context) { addNumberOfPagesAndSectionsToAnalyseToTrace(document.getNumberOfPages(), sectionsToAnalyze.size()); @@ -88,11 +95,14 @@ public class EntityDroolsExecutionService { .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession, nodesInKieSession); + RulesLogger logger = new RulesLogger(webSocketService); kieSession.setGlobal("document", document); kieSession.setGlobal("entityCreationService", entityCreationService); kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); kieSession.setGlobal("dictionary", dictionary); + kieSession.setGlobal("logger", logger); + kieSession.setGlobal("context", context); kieSession.insert(document); @@ -132,12 +142,14 @@ public class EntityDroolsExecutionService { completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS) .get(); } catch (ExecutionException e) { + logger.error(context, "Exception during rule execution", e); kieSession.dispose(); if (e.getCause() instanceof TimeoutException) { throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY); } throw new RuntimeException(e); } catch (InterruptedException e) { + logger.error(context, "Exception during rule execution", e); kieSession.dispose(); throw new RuntimeException(e); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml index 0f9a07a7..3976b359 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml @@ -16,6 +16,14 @@ project.version: 1.0-SNAPSHOT server: port: 8080 +fforesight: + keycloak: + enabled: true + ignored-endpoints: [ '/redaction-gateway-v1', '/actuator/health/**',"/api/rules-logging/rulesocket","/api/rules-logging/rulesocket/**", '/redaction-gateway-v1/async/download/with-ott/**', + '/internal-api/**', '/redaction-gateway-v1/docs/swagger-ui', + '/redaction-gateway-v1/docs/**','/redaction-gateway-v1/docs', + '/api', '/api/','/api/docs/**','/api/docs','/api/docs/swagger-ui' ] + spring: application: name: redaction-service diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl index beabfd0c..75152656 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; @@ -52,6 +54,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl index d55381be..560536a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl @@ -12,8 +12,12 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; + import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService; import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; @@ -26,6 +30,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; global ComponentCreationService componentCreationService +global ComponentMappingService componentMappingService +global RulesLogger logger +global Context context /** The imports, globals, queries and rules from this file are required for any component rule file. diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl index a61c62e5..8819c93c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -59,6 +61,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context /** The imports, globals, queries and rules from this file are required for any entity rule file. diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 4e1f484b..00407404 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -59,6 +59,7 @@ import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver; import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor; import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; @@ -191,6 +192,9 @@ public abstract class AbstractRedactionIntegrationTest { @MockBean protected DictionaryClient dictionaryClient; + @MockBean + protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; + @BeforeEach public void setup() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index 9be7e540..b2ba1e97 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -15,6 +15,7 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.kie.api.runtime.KieContainer; +import org.mockito.Mock; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -38,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantsClient; @@ -55,6 +57,9 @@ public class DictionaryServiceTest { @MockBean protected KieContainer kieContainer; + @MockBean + protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; + @MockBean protected DictionaryClient dictionaryClient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 0f4352b4..f30e6edf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -87,6 +87,7 @@ import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilitie import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; @@ -253,6 +254,8 @@ public class RulesTest { private LegalBasisClient legalBasisClient; @MockBean private TenantsClient tenantsClient; + @MockBean + private TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; @BeforeEach diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 8692f648..c13e3e19 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -34,6 +34,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; @@ -154,12 +155,14 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart); long droolsStart = System.currentTimeMillis(); + Context context = new Context("fileId", "dossierId", "dossierTemplateId", 1, 1, "redaction"); List fileAttributes = entityDroolsExecutionService.executeRules(kieContainer, document, dictionary, Collections.emptyList(), new ManualRedactions(), - new NerEntities()); + new NerEntities(), + context); System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart); System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/logger/RulesLoggerTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/logger/RulesLoggerTest.java new file mode 100644 index 00000000..3a01498f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/logger/RulesLoggerTest.java @@ -0,0 +1,85 @@ +package com.iqser.red.service.redaction.v1.server.logger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import org.drools.io.ClassPathResource; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.kie.api.KieServices; +import org.kie.api.builder.KieBuilder; +import org.kie.api.builder.KieFileSystem; +import org.kie.api.builder.ReleaseId; +import org.kie.api.runtime.KieContainer; +import org.kie.api.runtime.KieSession; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.iqser.red.service.redaction.v1.server.service.WebSocketService; + +@ExtendWith(MockitoExtension.class) +class RulesLoggerTest { + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + + @Mock + private WebSocketService webSocketService; + + @InjectMocks + private RulesLogger logger; + + + @Test + void testRulesLogging() { + + System.setOut(new PrintStream(outContent)); + RulesLogger logger = new RulesLogger(webSocketService); + + KieServices ks = KieServices.Factory.get(); + KieFileSystem kfs = ks.newKieFileSystem(); + + kfs.write(new ClassPathResource("logs/rules_logging.drl")); + + KieBuilder kieBuilder = ks.newKieBuilder(kfs).buildAll(); + ReleaseId releaseId = kieBuilder.getKieModule().getReleaseId(); + KieContainer kContainer = ks.newKieContainer(releaseId); + KieSession kSession = kContainer.newKieSession(); + + Context context = new Context("fileId", "dossierId", "dossierTemplateId", 1, 1, "redaction"); + kSession.setGlobal("logger", logger); + kSession.setGlobal("context", context); + + try { + kSession.fireAllRules(); + } catch (Exception e) { + logger.error(context, "Exception during rule execution", e); + } finally { + kSession.dispose(); + } + + System.setOut(originalOut); + System.out.println(outContent); + ArgumentCaptor argument = ArgumentCaptor.forClass(RuleLogEvent.class); + verify(webSocketService, times(3)).sendLogEvent(argument.capture()); + assertEquals(argument.getAllValues().get(0).getLogLevel(), LogLevel.INFO); + assertEquals(argument.getAllValues().get(0).getRuleVersion(), 1); + assertEquals(argument.getAllValues().get(0).getTenantId(), "redaction"); + assertEquals(argument.getAllValues().get(0).getFileId(), "fileId"); + assertEquals(argument.getAllValues().get(0).getDossierId(), "dossierId"); + assertEquals(argument.getAllValues().get(0).getAnalysisNumber(), 1); + assertEquals(argument.getAllValues().get(0).getDossierTemplateId(), "dossierTemplateId"); + assertEquals(argument.getAllValues().get(0).getMessage(), "This is a test log placeholder"); + assertEquals(argument.getAllValues().get(1).getLogLevel(), LogLevel.WARN); + assertEquals(argument.getAllValues().get(1).getMessage(), "This is a warning log with multiple placeholders p1 p2 p3"); + assertEquals(argument.getAllValues().get(2).getLogLevel(), LogLevel.ERROR); + assertEquals(argument.getAllValues().get(2).getMessage(), "Exception during rule execution Exception: Exception executing consequence for rule \"LOG.0.2: Test log error\" in drools: java.lang.NullPointerException: Cannot invoke \"String.toString()\" because \"result\" is null"); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java index 8cc15736..365dbeac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java @@ -56,6 +56,7 @@ import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantsClient; @@ -91,6 +92,9 @@ public class LiveDataIntegrationTest { @MockBean private LegalBasisClient legalBasisClient; + @MockBean + protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; + @Autowired private ResourcePatternResolver resourcePatternResolver; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 1241a76c..7909714a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 47d66f3d..d1220cb9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 77214ed4..778ef6fa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; @@ -52,6 +54,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl index dea2834f..9ba49e3f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl @@ -12,6 +12,9 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; + import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService; @@ -28,6 +31,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu global ComponentCreationService componentCreationService global ComponentMappingService componentMappingService +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index 1c70a936..0fe0c5a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 4431672c..be1b4544 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 02ef9031..3ba52b73 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 614b2b49..dceccd0b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 235469eb..6180b70d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl index d8b84772..1c3acb74 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl @@ -12,6 +12,9 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; + import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; @@ -26,6 +29,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; global ComponentCreationService componentCreationService +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl index 72b65851..0361a3f4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl @@ -12,6 +12,9 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; + import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; @@ -26,6 +29,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; global ComponentCreationService componentCreationService +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 6dc5c515..8e72fe89 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/logs/rules_logging.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/logs/rules_logging.drl new file mode 100644 index 00000000..49ef4924 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/logs/rules_logging.drl @@ -0,0 +1,32 @@ +package drools + +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; + +global RulesLogger logger +global Context context + + +rule "LOG.0.0: Test log info" +salience 1 + when + eval(true) + then + logger.info(context, "This is a test log {}", "placeholder"); + end + +rule "LOG.0.1: Test log warn" +salience 1 + when + eval(true) + then + logger.warn(context, "This is a warning log with multiple placeholders {} {} {}", "p1", "p2", "p3"); + end + +rule "LOG.0.2: Test log error" + when + eval(true) + then + String result = null; + result.toString(); + end \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index d1c66b0f..8b7cb1ba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.document.graph.*; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; @@ -55,6 +57,8 @@ global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global NerEntitiesAdapter nerEntitiesAdapter global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 11d9441c..827e2710 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; @@ -60,6 +62,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------ diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index beabfd0c..75152656 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -12,6 +12,8 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; +import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; @@ -52,6 +54,8 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary +global RulesLogger logger +global Context context //------------------------------------ queries ------------------------------------