From aad890b8930fd888b6235f27df50809305b4b2a6 Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Thu, 7 Mar 2024 14:38:13 +0100 Subject: [PATCH] RED-7700: Safe rule execution --- .../v1/model/DroolsBlacklistErrorMessage.java | 27 ++ .../model/DroolsSyntaxDeprecatedWarnings.java | 9 +- .../v1/model/DroolsSyntaxErrorMessage.java | 9 +- .../v1/model/DroolsSyntaxValidation.java | 37 -- .../redaction/v1/model/DroolsValidation.java | 39 ++ .../v1/model/DroolsValidationMessage.java | 21 + .../v1/resources/RedactionResource.java | 4 +- .../v1/server/RedactionServiceSettings.java | 2 + .../controller/RedactionController.java | 10 +- .../model/drools/RuleFileBluePrint.java | 4 +- ...vice.java => DroolsValidationService.java} | 135 ++++-- .../server/service/drools/RuleFileParser.java | 33 +- .../storage/RuleManagementResources.java | 16 + .../src/main/resources/application-dev.yaml | 1 + .../src/main/resources/drools/blacklist.txt | 17 + .../DroolsSyntaxValidationServiceTest.java | 344 --------------- .../services/DroolsValidationServiceTest.java | 400 ++++++++++++++++++ 17 files changed, 659 insertions(+), 449 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsBlacklistErrorMessage.java delete mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidation.java create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidationMessage.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/{DroolsSyntaxValidationService.java => DroolsValidationService.java} (66%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/blacklist.txt delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsBlacklistErrorMessage.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsBlacklistErrorMessage.java new file mode 100644 index 00000000..5a8f22a2 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsBlacklistErrorMessage.java @@ -0,0 +1,27 @@ +package com.iqser.red.service.redaction.v1.model; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@AllArgsConstructor +@NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(callSuper = true) +public class DroolsBlacklistErrorMessage extends DroolsValidationMessage { + + List blacklistedKeywords; + + public String getMessage() { + return String.format("Blacklisted keywords found in this rule: %s", String.join(", ", blacklistedKeywords)); + } + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java index 08e0e65b..c63b6986 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java @@ -4,18 +4,19 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.experimental.SuperBuilder; @Data -@Builder +@SuperBuilder @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class DroolsSyntaxDeprecatedWarnings { +@EqualsAndHashCode(callSuper = true) +public class DroolsSyntaxDeprecatedWarnings extends DroolsValidationMessage { - Integer line; - Integer column; String message; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java index e6312466..8a88d5b7 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java @@ -4,18 +4,19 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.experimental.SuperBuilder; @Data -@Builder +@SuperBuilder @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class DroolsSyntaxErrorMessage { +@EqualsAndHashCode(callSuper = true) +public class DroolsSyntaxErrorMessage extends DroolsValidationMessage { - Integer line; - Integer column; String message; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java deleted file mode 100644 index 3d35c2a3..00000000 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.iqser.red.service.redaction.v1.model; - -import java.util.LinkedList; -import java.util.List; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class DroolsSyntaxValidation { - - @Builder.Default - List droolsSyntaxErrorMessages = new LinkedList<>(); - @Builder.Default - List droolsSyntaxDeprecatedWarnings = new LinkedList<>(); - - - public void addErrorMessage(int line, int column, String message) { - - getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build()); - } - - - public boolean isCompiled() { - - return droolsSyntaxErrorMessages.isEmpty(); - } - -} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidation.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidation.java new file mode 100644 index 00000000..988413ae --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidation.java @@ -0,0 +1,39 @@ +package com.iqser.red.service.redaction.v1.model; + +import java.util.ArrayList; +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class DroolsValidation { + + @Builder.Default + List syntaxErrorMessages = new ArrayList<>(); + @Builder.Default + List deprecatedWarnings = new ArrayList<>(); + @Builder.Default + List blacklistErrorMessages = new ArrayList<>(); + + + public void addErrorMessage(int line, int column, String message) { + + getSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build()); + } + + + public boolean isCompiled() { + + return syntaxErrorMessages.isEmpty() && blacklistErrorMessages.isEmpty(); + } + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidationMessage.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidationMessage.java new file mode 100644 index 00000000..88948f9a --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidationMessage.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@AllArgsConstructor +@NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class DroolsValidationMessage { + + Integer line; + Integer column; +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index ffdb7160..ceba12fa 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -4,12 +4,12 @@ import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.DroolsValidation; import com.iqser.red.service.redaction.v1.model.RuleValidationModel; public interface RedactionResource { @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE) - DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel); + DroolsValidation testRules(@RequestBody RuleValidationModel rulesValidationModel); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index 2cc7f8f4..7accb9d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -30,4 +30,6 @@ public class RedactionServiceSettings { private int droolsExecutionTimeoutSecs = 300; + private boolean ruleExecutionSecured = true; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index bd9c9951..2b47ceed 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -3,10 +3,10 @@ package com.iqser.red.service.redaction.v1.server.controller; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.DroolsValidation; import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RedactionResource; -import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; @@ -17,14 +17,14 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class RedactionController implements RedactionResource { - private final DroolsSyntaxValidationService droolsSyntaxValidationService; + private final DroolsValidationService droolsValidationService; @Override - public DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel) { + public DroolsValidation testRules(@RequestBody RuleValidationModel rulesValidationModel) { try { - return droolsSyntaxValidationService.testRules(rulesValidationModel); + return droolsValidationService.testRules(rulesValidationModel); } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java index 5f98935c..338e58a7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java @@ -10,7 +10,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.DroolsValidation; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -32,7 +32,7 @@ public final class RuleFileBluePrint { int globalsLine; List queries; List ruleClasses; - DroolsSyntaxValidation droolsSyntaxValidation; + DroolsValidation droolsValidation; public Optional findRuleClassByType(RuleType ruleType) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java similarity index 66% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index 1e1fe0ef..392dfab9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.drools.drl.parser.DroolsParserException; import org.kie.api.builder.KieBuilder; @@ -15,11 +16,13 @@ import org.springframework.stereotype.Service; import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.redaction.v1.model.DroolsBlacklistErrorMessage; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxDeprecatedWarnings; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage; -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.DroolsValidation; import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; @@ -35,29 +38,58 @@ import lombok.extern.slf4j.Slf4j; @Service @RequiredArgsConstructor @Slf4j -public class DroolsSyntaxValidationService { - - private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$"); +public class DroolsValidationService { + private final RedactionServiceSettings redactionServiceSettings; private final KieContainerCreationService kieContainerCreationService; - private final DeprecatedElementsFinder deprecatedElementsFinder; + private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$"); + public static final String LINEBREAK_MATCHER = "\\R"; @SneakyThrows - public DroolsSyntaxValidation testRules(RuleValidationModel rules) { + public DroolsValidation testRules(RuleValidationModel rules) { - DroolsSyntaxValidation customDroolsSyntaxValidation; + DroolsValidation customDroolsValidation; try { - customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType())); + customDroolsValidation = buildCustomDroolsValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType())); } catch (DroolsParserException e) { // this means the parser could not parse the file at all. In this case use drools compiler only as it will return useful error messages. - customDroolsSyntaxValidation = new DroolsSyntaxValidation(); + customDroolsValidation = new DroolsValidation(); } - DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules); - droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages()); - droolsCompilerSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(customDroolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings()); - return droolsCompilerSyntaxValidation; + DroolsValidation droolsCompilerValidation = buildDroolsCompilerValidation(rules); + droolsCompilerValidation.getSyntaxErrorMessages().addAll(customDroolsValidation.getSyntaxErrorMessages()); + droolsCompilerValidation.getDeprecatedWarnings().addAll(customDroolsValidation.getDeprecatedWarnings()); + droolsCompilerValidation.getBlacklistErrorMessages().addAll(customDroolsValidation.getBlacklistErrorMessages()); + return droolsCompilerValidation; + } + + + private DroolsValidation buildCustomDroolsValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + + DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation(); + + addSyntaxDeprecatedWarnings(ruleFileBluePrint, customValidation); + + addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation); + + if (redactionServiceSettings.isRuleExecutionSecured()) { + addBlacklistErrorMessages(ruleFileBluePrint, customValidation); + } + + return customValidation; + } + + + private void addSyntaxDeprecatedWarnings(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + // find deprecated elements in the ruleFileBluePrint + DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); + if (warningMessageForImports != null) { + customValidation.getDeprecatedWarnings().add(warningMessageForImports); + } + customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } @@ -111,18 +143,7 @@ public class DroolsSyntaxValidationService { } - private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { - - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); - - DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation(); - - // find deprecated elements in the ruleFileBluePrint - DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); - if (warningMessageForImports != null) { - customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().add(warningMessageForImports); - } - customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); + private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); @@ -130,7 +151,7 @@ public class DroolsSyntaxValidationService { }; if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) { - customSyntaxValidation.getDroolsSyntaxErrorMessages() + customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(ruleFileBluePrint.getImportLine()) .column(0) @@ -138,7 +159,7 @@ public class DroolsSyntaxValidationService { .build()); } if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) { - customSyntaxValidation.getDroolsSyntaxErrorMessages() + customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(ruleFileBluePrint.getGlobalsLine()) .column(0) @@ -148,7 +169,7 @@ public class DroolsSyntaxValidationService { baseRuleFileBluePrint.getQueries() .forEach(basicQuery -> { if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { - customSyntaxValidation.getDroolsSyntaxErrorMessages() + customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(basicQuery.getLine()) .column(0) @@ -159,7 +180,7 @@ public class DroolsSyntaxValidationService { if (ruleFileType.equals(RuleFileType.ENTITY)) { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { - customSyntaxValidation.getDroolsSyntaxErrorMessages() + customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(0) .column(0) @@ -167,7 +188,6 @@ public class DroolsSyntaxValidationService { .build()); } } - return customSyntaxValidation; } @@ -199,7 +219,54 @@ public class DroolsSyntaxValidationService { } - private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(RuleValidationModel rules) { + private void addBlacklistErrorMessages(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + + List blacklistErrorMessages = new ArrayList<>(); + + List blacklistedKeywords = parseBlacklistFile(RuleManagementResources.getBlacklistFileString()); + + // checks the rules for occurrence of blacklisted keyword + if (!blacklistedKeywords.isEmpty()) { + SearchImplementation blacklistedKeywordSearchImplementation = new SearchImplementation(blacklistedKeywords, false); + + for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { + for (BasicRule basicRule : ruleUnit.rules()) { + List matches = blacklistedKeywordSearchImplementation.getMatches(basicRule.getCode()); + + if (!matches.isEmpty()) { + List foundBlacklistedKeywords = matches.stream() + .map(m -> basicRule.getCode().substring(m.startIndex(), m.endIndex())) + .distinct() + .toList(); + blacklistErrorMessages.add(DroolsBlacklistErrorMessage.builder() + .line(basicRule.getLine()) + .column(0) + .blacklistedKeywords(foundBlacklistedKeywords) + .build()); + } + } + } + } + } + + customValidation.getBlacklistErrorMessages() + .addAll(blacklistErrorMessages.stream() + .sorted(Comparator.comparingInt(DroolsBlacklistErrorMessage::getLine)) + .toList()); + } + + + private List parseBlacklistFile(String blacklistFileString) { + + return Stream.of(blacklistFileString.split(LINEBREAK_MATCHER)) + .distinct() + .filter(s -> !s.isBlank()) + .toList(); + } + + + private DroolsValidation buildDroolsCompilerValidation(RuleValidationModel rules) { var versionId = System.currentTimeMillis(); var testRules = "test-rules"; @@ -207,17 +274,17 @@ public class DroolsSyntaxValidationService { versionId, rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType())); - return buildDroolsCompilerSyntaxValidation(kieBuilder); + return buildDroolsCompilerValidation(kieBuilder); } - private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) { + private DroolsValidation buildDroolsCompilerValidation(KieBuilder kieBuilder) { List errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR); List droolsSyntaxErrorMessages = errorMessages.stream() .map(this::buildDroolsSyntaxErrorMessage) .collect(Collectors.toList()); - return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build(); + return DroolsValidation.builder().syntaxErrorMessages(droolsSyntaxErrorMessages).build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java index 0c4ed5f8..c50c2acc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java @@ -16,7 +16,7 @@ import org.drools.drl.ast.descr.RuleDescr; import org.drools.drl.parser.DrlParser; import org.kie.internal.builder.conf.LanguageLevelOption; -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.DroolsValidation; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; @@ -38,7 +38,7 @@ public class RuleFileParser { @SneakyThrows public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { - DroolsSyntaxValidation customDroolsSyntaxValidation = DroolsSyntaxValidation.builder().build(); + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); PackageDescr packageDescr = parser.parse(false, ruleString); List allRules = new LinkedList<>(); @@ -48,7 +48,7 @@ public class RuleFileParser { if (rule.isQuery()) { allQueries.add(new BasicQuery(rule.getName(), rule.getLine(), ruleString.substring(rule.getStartCharacter(), rule.getEndCharacter()))); } else { - validateRule(ruleString, rule, customDroolsSyntaxValidation, allRules); + validateRule(ruleString, rule, customDroolsValidation, allRules); } } @@ -78,29 +78,28 @@ public class RuleFileParser { .map(GlobalDescr::getLine) .orElse(0), allQueries, - ruleClasses, - customDroolsSyntaxValidation); + ruleClasses, customDroolsValidation); } - private static void validateRule(String ruleString, RuleDescr rule, DroolsSyntaxValidation customDroolsSyntaxValidation, List allRules) { + private static void validateRule(String ruleString, RuleDescr rule, DroolsValidation customDroolsValidation, List allRules) { BasicRule basicRule; try { basicRule = BasicRule.fromRuleDescr(rule, ruleString); } catch (Exception e) { - customDroolsSyntaxValidation.addErrorMessage(rule.getLine(), rule.getColumn(), "Malformed rule name, correct format is \"\\w+.\\d+.\\d+: \""); + customDroolsValidation.addErrorMessage(rule.getLine(), rule.getColumn(), "Malformed rule name, correct format is \"\\w+.\\d+.\\d+: \""); return; } if (allRules.contains(basicRule)) { - addDuplicateRuleIdentifierErrorMessage(rule, basicRule, customDroolsSyntaxValidation); + addDuplicateRuleIdentifierErrorMessage(rule, basicRule, customDroolsValidation); } - validateRuleIdentifierInCodeIsSame(basicRule.getCode(), basicRule.getIdentifier().toString(), rule.getLine(), customDroolsSyntaxValidation); + validateRuleIdentifierInCodeIsSame(basicRule.getCode(), basicRule.getIdentifier().toString(), rule.getLine(), customDroolsValidation); allRules.add(BasicRule.fromRuleDescr(rule, ruleString)); } - private static void validateRuleIdentifierInCodeIsSame(String code, String identifier, int lineOffset, DroolsSyntaxValidation customDroolsSyntaxValidation) { + private static void validateRuleIdentifierInCodeIsSame(String code, String identifier, int lineOffset, DroolsValidation customDroolsValidation) { Matcher matcher = ruleIdentifierInCodeFinder.matcher(code); while (matcher.find()) { @@ -108,19 +107,19 @@ public class RuleFileParser { long line = code.substring(0, matcher.start(1)).lines() .count() + lineOffset - 1; if (!identifier.equals(identifierInCode)) { - customDroolsSyntaxValidation.addErrorMessage((int) line, - 0, - String.format("Rule identifier %s is not equal to rule identifier %s in rule name!", identifierInCode, identifier)); + customDroolsValidation.addErrorMessage((int) line, + 0, + String.format("Rule identifier %s is not equal to rule identifier %s in rule name!", identifierInCode, identifier)); } } } - private void addDuplicateRuleIdentifierErrorMessage(RuleDescr rule, BasicRule basicRule, DroolsSyntaxValidation customDroolsSyntaxValidation) { + private void addDuplicateRuleIdentifierErrorMessage(RuleDescr rule, BasicRule basicRule, DroolsValidation customDroolsValidation) { - customDroolsSyntaxValidation.addErrorMessage(rule.getLine(), - rule.getColumn(), - String.format("RuleIdentifier: %s is a duplicate, duplicates are not allowed!", basicRule.getIdentifier())); + customDroolsValidation.addErrorMessage(rule.getLine(), + rule.getColumn(), + String.format("RuleIdentifier: %s is a duplicate, duplicates are not allowed!", basicRule.getIdentifier())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java index e320ae7a..0089da9c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java @@ -43,4 +43,20 @@ public class RuleManagementResources { } } + + @SneakyThrows + public static InputStream getBlacklistFileInputStream() { + + return new ClassPathResource(Path.of(folderPrefix, "blacklist.txt").toString()).getInputStream(); + } + + + @SneakyThrows + public static String getBlacklistFileString() { + + try (var in = getBlacklistFileInputStream()) { + return new String(in.readAllBytes()); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml index d9bb2f18..4df10cea 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml @@ -16,6 +16,7 @@ redaction-service: cvTableParsingEnabled: false nerServiceEnabled: false priorityMode: false + ruleExecutionSecured: false application: type: "RedactManager" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/blacklist.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/blacklist.txt new file mode 100644 index 00000000..62d61d1f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/blacklist.txt @@ -0,0 +1,17 @@ +System. +Runtime. +Thread. +ProcessBuilder. +SecurityManager. +ClassLoader. +Class. + +java.io.File +java.nio.file +java.io.Object + +java.net + +java.lang + +java.util.zip diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java deleted file mode 100644 index 48d612a6..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java +++ /dev/null @@ -1,344 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.drools.files.management.services; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.FileOutputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.List; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.MockitoAnnotations; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.core.io.ClassPathResource; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; -import com.iqser.red.service.redaction.v1.model.RuleValidationModel; -import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; -import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService; -import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; -import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; - -import lombok.SneakyThrows; - -class DroolsSyntaxValidationServiceTest { - - @MockBean - RulesClient rulesClient; - @MockBean - EntityEnrichmentService entityEnrichmentService; - - - @BeforeEach - public void setupMocks() { - - MockitoAnnotations.openMocks(this); - } - - - @Test - @SneakyThrows - void testRules() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testRulesWithRemovedImportsAndChangedName() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", ""); - rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\""); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testRulesWithAddedImports() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", - "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;"); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testDocumineRules() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/documine_flora.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testCorruptedRules() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - String corruptedRules = rulesString.replaceAll(";", ""); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); - assertFalse(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testCorruptedImports() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;", - "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;"); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); - assertFalse(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testRemoveQuery() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - String corruptedRules = rulesString.replaceAll("query \"getFileAttributes\"\n.*\n *end", ""); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertFalse(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testComponentDrools() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/documine_flora_components.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.COMPONENT.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testComponentDroolsAsEntityDrools() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/documine_flora_components.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertFalse(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - @Disabled - void attemptImportsFixToAllRuleFiles() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - - List ruleFiles = List.of("drools/rules.drl", - "drools/all_redact_manager_rules.drl", - "drools/documine_flora.drl", - "drools/manual_redaction_rules.drl", - "drools/acceptance_rules.drl", - "drools/rules_v2.drl"); - - for (String ruleFile : ruleFiles) { - var rulesFile = new ClassPathResource(ruleFile); - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - if (droolsSyntaxValidation.isCompiled()) { - continue; - } - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); - - rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); - rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); - - try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { - outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); - } - - } - - } - - - @Test - @SneakyThrows - void testRulesWithUnallowedAddedImports() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", - "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;"); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertFalse(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testAllRules() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testAcceptanceRules() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/acceptance_rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testManualRedactionRules() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testRulesV2() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules_v2.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testRulesWithDeprecatedFunctions() { - - DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient), - new DeprecatedElementsFinder()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - rulesString = rulesString.replaceAll(".resize\\(", ".resizeEntityAndReinsert("); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); - droolsSyntaxValidation.getDroolsSyntaxErrorMessages() - .forEach(System.out::println); - assertTrue(droolsSyntaxValidation.isCompiled()); - assertEquals(droolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().size(), 2); - } - -} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java new file mode 100644 index 00000000..09a04a6d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -0,0 +1,400 @@ +package com.iqser.red.service.redaction.v1.server.drools.files.management.services; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.mockito.MockitoAnnotations; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.redaction.v1.model.DroolsValidation; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; +import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; +import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; + +import lombok.SneakyThrows; + +class DroolsValidationServiceTest { + + @MockBean + RulesClient rulesClient; + @MockBean + EntityEnrichmentService entityEnrichmentService; + @MockBean + RedactionServiceSettings redactionServiceSettings; + + + @BeforeEach + public void setupMocks() { + + MockitoAnnotations.openMocks(this); + } + + + @Test + @SneakyThrows + void testRules() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testRulesWithRemovedImportsAndChangedName() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", ""); + rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\""); + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testRulesWithAddedImports() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", + "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;"); + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testDocumineRules() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/documine_flora.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testCorruptedRules() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + String corruptedRules = rulesString.replaceAll(";", ""); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); + assertFalse(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testCorruptedImports() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;", + "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;"); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); + assertFalse(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testRemoveQuery() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + String corruptedRules = rulesString.replaceAll("query \"getFileAttributes\"\n.*\n *end", ""); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertFalse(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testComponentDrools() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/documine_flora_components.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.COMPONENT.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testComponentDroolsAsEntityDrools() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/documine_flora_components.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertFalse(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + @Disabled + void attemptImportsFixToAllRuleFiles() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + + List ruleFiles = List.of("drools/rules.drl", + "drools/all_redact_manager_rules.drl", + "drools/documine_flora.drl", + "drools/manual_redaction_rules.drl", + "drools/acceptance_rules.drl", + "drools/rules_v2.drl"); + + for (String ruleFile : ruleFiles) { + var rulesFile = new ClassPathResource(ruleFile); + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + if (droolsValidation.isCompiled()) { + continue; + } + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + + rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); + rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + + try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { + outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); + } + + } + + } + + + @Test + @SneakyThrows + void testRulesWithUnallowedAddedImports() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", + "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;"); + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertFalse(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testAllRules() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testAcceptanceRules() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/acceptance_rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testManualRedactionRules() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testRulesV2() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules_v2.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testRulesWithDeprecatedFunctions() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + rulesString = rulesString.replaceAll(".resize\\(", ".resizeEntityAndReinsert("); + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getSyntaxErrorMessages() + .forEach(System.out::println); + assertTrue(droolsValidation.isCompiled()); + assertEquals(droolsValidation.getDeprecatedWarnings().size(), 2); + } + + + + @Test + @SneakyThrows + void testRulesWithBlacklistedKeyword() { + + DroolsValidationService droolsValidationService = new DroolsValidationService(new RedactionServiceSettings(), + new KieContainerCreationService(rulesClient), + new DeprecatedElementsFinder()); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + String evilRule = """ + + //------------------------------------ All the evil rules ------------------------------------ + + // Rule unit: EV.1 + rule "EV.1.0: Remove duplicate FileAttributes but also do very evil things" + salience 999 + when + $fileAttribute: FileAttribute($label: label, $value: value) + $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value) + then + retract($duplicate); + System.exit(0); + end + """; + rulesString = rulesString + evilRule; + + DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsValidation.getBlacklistErrorMessages() + .forEach(System.out::println); + assertFalse(droolsValidation.isCompiled()); + assertEquals(droolsValidation.getBlacklistErrorMessages().size(), 1); + } + +} \ No newline at end of file