From fb8f227ac9a5f82899dc7c3ee2a7c7b46078ed86 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Sep 2024 10:24:59 +0200 Subject: [PATCH 1/9] RED-9472: seperation of system rules added endpoints for seperating and readding system rules working on service --- .../redaction-service-api-v1/build.gradle.kts | 1 + .../v1/resources/RuleBuilderResource.java | 11 ++++ .../build.gradle.kts | 1 + .../controller/RuleBuilderController.java | 29 ++++++++-- .../v1/server/model/RulesResponse.java | 23 ++++++++ .../model/drools/RuleFileBluePrint.java | 14 +++++ .../v1/server/service/RuleBuilderService.java | 55 +++++++++++++++++++ .../server/service/drools/RuleFileParser.java | 53 ++++++++++++++++++ 8 files changed, 181 insertions(+), 6 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RulesResponse.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 1f228316..3bf38721 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -9,6 +9,7 @@ val persistenceServiceVersion = "2.545.0" dependencies { implementation("org.springframework:spring-web:6.0.12") implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + api("com.knecon.fforesight:swagger-commons:0.7.0") } publishing { diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index 3864ba9b..5f0563f1 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,13 +1,24 @@ package com.iqser.red.service.redaction.v1.resources; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.multipart.MultipartFile; + +import io.swagger.v3.oas.annotations.media.Schema; public interface RuleBuilderResource { @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE) RuleBuilderModel getRuleBuilderModel(); + @PostMapping(value="/rulefile-system", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithoutSystemRules(@Schema(type = "string", format = "UTF_8", name = "file") @RequestPart(name = "ruleFile") String ruleFile); + + @PostMapping(value="/rulefile", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithSeperatedSystemRules(@Schema(type = "string", format = "UTF_8", name = "file") @RequestPart(name = "ruleFile") String ruleFile); + } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 87c4b538..a7061434 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -76,6 +76,7 @@ dependencies { implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") + api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 306dccbe..f690127f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,27 +1,44 @@ package com.iqser.red.service.redaction.v1.server.controller; -import java.util.Collections; - import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @RestController +@Slf4j @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { + private final RuleBuilderService ruleBuilderService; + + @Override public RuleBuilderModel getRuleBuilderModel() { - RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + return this.ruleBuilderService.getRuleBuilderModel(); + } - ruleBuilderModel.setWhenClauses(Collections.emptyList()); - ruleBuilderModel.setThenConditions(Collections.emptyList()); - return ruleBuilderModel; + @Override + public RulesResponse getRuleFileWithoutSystemRules(String rulesString) { + + RulesResponse rulesResponse = new RulesResponse(); + String filteredRules = this.ruleBuilderService.getRuleFileWithoutSystemRules(rulesString); + log.info("Filtered rules: {}", filteredRules); + return rulesResponse; + } + + + @Override + public RulesResponse getRuleFileWithSeperatedSystemRules(String rulesString) { + + return this.ruleBuilderService.getRuleFileWithSeperatedSystemRules(rulesString); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RulesResponse.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RulesResponse.java new file mode 100644 index 00000000..442e249e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RulesResponse.java @@ -0,0 +1,23 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Schema(description = "Object containing a string of Drools rules.") +public class RulesResponse { + + @Schema(description = "The actual string of rules.") + private String rules; + + @Schema(description = "The DossierTemplate Id for these rules") + private String dossierTemplateId; + + @Schema(description = "Bad written rules can lead to timeouts or endless processing. This will be detected by the system and all analyse request for the rules will be rejected. This flag indicates that a timeout was detected and you need to fix the rules") + private boolean timeoutDetected; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java index 338e58a7..98d6cf0a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java @@ -18,8 +18,10 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Builder @NoArgsConstructor @AllArgsConstructor @@ -43,6 +45,18 @@ public final class RuleFileBluePrint { } + public void dropRulesByIdentifier(RuleType ruleType) { + + log.info("removing rule with identifier {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .collect(Collectors.toList()); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + public Set getImportSplitByKeyword() { return Arrays.stream(imports.replaceAll("\n", "").split("import")) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java new file mode 100644 index 00000000..c849802f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -0,0 +1,55 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class RuleBuilderService { + + private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + + + public RuleBuilderModel getRuleBuilderModel() { + + RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + + ruleBuilderModel.setWhenClauses(Collections.emptyList()); + ruleBuilderModel.setThenConditions(Collections.emptyList()); + + return ruleBuilderModel; + } + + + public String getRuleFileWithoutSystemRules(String rulesString) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + log.info("Starting to remove system rules from ruleFile {}", ruleFileBluePrint); + for (String systemRule : systemRules) { + ruleFileBluePrint.dropRulesByIdentifier(RuleType.fromString(systemRule)); + } + log.info("Finished removing system rules for ruleFile {}", ruleFileBluePrint); + + return RuleFileParser.buildRulesStringFromBluePrint(ruleFileBluePrint); + } + + + public RulesResponse getRuleFileWithSeperatedSystemRules(String rulesString) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + return null; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java index 375dc0f2..efd2976c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java @@ -83,6 +83,59 @@ public class RuleFileParser { } + @SneakyThrows + public String buildRulesStringFromBluePrint(RuleFileBluePrint bluePrint) { + + StringBuilder ruleStringBuilder = new StringBuilder(); + + // Append imports + ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); + + // Append globals + if (!bluePrint.getGlobals().isEmpty()) { + ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); + } + + // Append queries + for (BasicQuery query : bluePrint.getQueries()) { + ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); + } + + // Append rules + for (RuleClass ruleClass : bluePrint.getRuleClasses()) { + ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); + } + + // Return the final rule string + return ruleStringBuilder.toString().trim(); + } + + + private String buildQueryString(BasicQuery query) { + + return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; + } + + + private String buildRuleString(RuleClass ruleClass) { + + StringBuilder ruleBuilder = new StringBuilder(); + + // Use RuleType to distinguish between different rule types, if needed + ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); + + for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { + for (BasicRule rule : ruleUnit.rules()) { + // Assuming BasicRule has a method to retrieve the condition as a string + ruleBuilder.append(rule.getCode()).append("\n"); + } + } + ruleBuilder.append("end"); + + return ruleBuilder.toString(); + } + + private static void validateRule(String ruleString, RuleDescr rule, DroolsValidation customDroolsValidation, List allRules) { BasicRule basicRule; -- 2.47.2 From b42b5762ec471f155a7f68ff53b3362f9786f290 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 26 Sep 2024 12:33:41 +0200 Subject: [PATCH 2/9] RED-9472: seperation of system rules refactored endpoints. now removing system rules on download merging user rules with systemrules on upload --- .../v1/resources/RuleBuilderResource.java | 21 +++++---- .../controller/RuleBuilderController.java | 31 +++++++++--- .../model/drools/RuleFileBluePrint.java | 47 +++++++++++++++++++ .../v1/server/service/RuleBuilderService.java | 31 +++++++++--- .../server/service/drools/RuleFileParser.java | 2 + 5 files changed, 109 insertions(+), 23 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index 5f0563f1..a0711376 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,24 +1,25 @@ package com.iqser.red.service.redaction.v1.resources; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; -import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; - import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestPart; -import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.bind.annotation.RequestBody; -import io.swagger.v3.oas.annotations.media.Schema; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; public interface RuleBuilderResource { @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE) RuleBuilderModel getRuleBuilderModel(); - @PostMapping(value="/rulefile-system", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) - RulesResponse getRuleFileWithoutSystemRules(@Schema(type = "string", format = "UTF_8", name = "file") @RequestPart(name = "ruleFile") String ruleFile); - @PostMapping(value="/rulefile", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) - RulesResponse getRuleFileWithSeperatedSystemRules(@Schema(type = "string", format = "UTF_8", name = "file") @RequestPart(name = "ruleFile") String ruleFile); + @PostMapping(value = "/internal-api/rules/user-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithoutSystemRules(@RequestBody SystemRulesSeperationRequest systemRulesSeperationRequest); + + + @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithSeperatedSystemRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index f690127f..5a57e521 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -3,9 +3,15 @@ package com.iqser.red.service.redaction.v1.server.controller; import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -16,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; public class RuleBuilderController implements RuleBuilderResource { private final RuleBuilderService ruleBuilderService; + private final DroolsValidationService droolsValidationService; @Override @@ -26,19 +33,31 @@ public class RuleBuilderController implements RuleBuilderResource { @Override - public RulesResponse getRuleFileWithoutSystemRules(String rulesString) { + public RulesResponse getRuleFileWithoutSystemRules(SystemRulesSeperationRequest systemRulesSeperationRequest) { RulesResponse rulesResponse = new RulesResponse(); - String filteredRules = this.ruleBuilderService.getRuleFileWithoutSystemRules(rulesString); - log.info("Filtered rules: {}", filteredRules); + String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); + try { + droolsValidationService.testRules(new RuleValidationModel("ENTITY", filteredRules)); + } catch (Exception e) { + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + } + rulesResponse.setRules(filteredRules); return rulesResponse; } @Override - public RulesResponse getRuleFileWithSeperatedSystemRules(String rulesString) { - - return this.ruleBuilderService.getRuleFileWithSeperatedSystemRules(rulesString); + public RulesResponse getRuleFileWithSeperatedSystemRules(RulesUpdateRequest rulesUpdateRequest) { + RulesResponse rulesResponse = new RulesResponse(); + String mergedRules = ruleBuilderService.getRuleFileWithSeperatedSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + try { + droolsValidationService.testRules(new RuleValidationModel("ENTITY", mergedRules)); + } catch (Exception e) { + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + } + rulesResponse.setRules(mergedRules); + return rulesResponse; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java index 98d6cf0a..8ac3abcb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java @@ -57,6 +57,47 @@ public final class RuleFileBluePrint { } + public void dropAllRulesExceptSystemRules(List systemRules) { + + log.info("removing all rules except the system rules"); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) + .collect(Collectors.toList()); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropImports() { + + this.imports = ""; + } + + + public void dropQueries() { + + this.queries.clear(); + } + + + public int countRuleOccurences(RuleType ruleType) { + + log.info("counting occurences of files {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .collect(Collectors.toList()); + return rulesToBeRemoved.size(); + } + + + private boolean filterOutRule(RuleType ruleType, List filteredRules) { + + return !filteredRules.contains(ruleType.name()); + } + + public Set getImportSplitByKeyword() { return Arrays.stream(imports.replaceAll("\n", "").split("import")) @@ -108,4 +149,10 @@ public final class RuleFileBluePrint { return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']'; } + + public void addRuleClass(RuleClass ruleClass) { + + this.ruleClasses.add(ruleClass); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index c849802f..9e5f180e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -7,7 +7,6 @@ import java.util.List; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; @@ -33,23 +32,41 @@ public class RuleBuilderService { } - public String getRuleFileWithoutSystemRules(String rulesString) { + public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - log.info("Starting to remove system rules from ruleFile {}", ruleFileBluePrint); + log.info("Starting to remove system rules from ruleFile"); for (String systemRule : systemRules) { ruleFileBluePrint.dropRulesByIdentifier(RuleType.fromString(systemRule)); } - log.info("Finished removing system rules for ruleFile {}", ruleFileBluePrint); + ruleFileBluePrint.dropQueries(); + if (removeImports) { + ruleFileBluePrint.dropQueries(); + } + log.info("Finished removing system rules for ruleFile"); return RuleFileParser.buildRulesStringFromBluePrint(ruleFileBluePrint); } - public RulesResponse getRuleFileWithSeperatedSystemRules(String rulesString) { + public String getRuleFileWithSeperatedSystemRules(String existingRules, String updatedRules) { - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - return null; + RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); + ruleFileBluePrintExisting.dropAllRulesExceptSystemRules(systemRules); + RuleFileBluePrint ruleFileBluePrintUpdate = RuleFileParser.buildBluePrintFromRulesString(updatedRules); + for (String systemRule : systemRules) { + if (ruleFileBluePrintUpdate.countRuleOccurences(RuleType.fromString(systemRule)) > 0) { + throw new RuntimeException("System rules are not allowed in an update."); + + } + } + ruleFileBluePrintExisting.setImports(ruleFileBluePrintUpdate.getImports()); + ruleFileBluePrintUpdate.getRuleClasses() + .forEach(ruleClass -> { + ruleFileBluePrintExisting.addRuleClass(ruleClass); + }); + + return RuleFileParser.buildRulesStringFromBluePrint(ruleFileBluePrintExisting); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java index efd2976c..c16f179c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java @@ -203,4 +203,6 @@ public class RuleFileParser { .collect(Collectors.toList()); } + private List getImportsFromRuleFile(String ruleString) + } -- 2.47.2 From 1ccdcf584475f4a512ec04ae2e5f9e6f6ae5e9fe Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 26 Sep 2024 12:34:01 +0200 Subject: [PATCH 3/9] RED-9472: seperation of system rules --- .../redaction/v1/server/service/drools/RuleFileParser.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java index c16f179c..efd2976c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java @@ -203,6 +203,4 @@ public class RuleFileParser { .collect(Collectors.toList()); } - private List getImportsFromRuleFile(String ruleString) - } -- 2.47.2 From 4978259fc66a528bef5e62a9fa359c0c045ddda5 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 30 Sep 2024 12:12:07 +0200 Subject: [PATCH 4/9] RED-9472: seperation of system rules --- .../server/controller/RuleBuilderController.java | 16 +++++++++++----- .../v1/server/service/RuleBuilderService.java | 6 ++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 5a57e521..09be0f0c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -8,7 +8,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; @@ -38,9 +37,9 @@ public class RuleBuilderController implements RuleBuilderResource { RulesResponse rulesResponse = new RulesResponse(); String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); try { - droolsValidationService.testRules(new RuleValidationModel("ENTITY", filteredRules)); + droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), filteredRules)); } catch (Exception e) { - throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + throw new RulesValidationException("Error trying to compile the cleaned rule file: " + filteredRules + " with message: {} " + e.getMessage(), e); } rulesResponse.setRules(filteredRules); return rulesResponse; @@ -49,15 +48,22 @@ public class RuleBuilderController implements RuleBuilderResource { @Override public RulesResponse getRuleFileWithSeperatedSystemRules(RulesUpdateRequest rulesUpdateRequest) { + RulesResponse rulesResponse = new RulesResponse(); String mergedRules = ruleBuilderService.getRuleFileWithSeperatedSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); try { - droolsValidationService.testRules(new RuleValidationModel("ENTITY", mergedRules)); + droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergedRules)); } catch (Exception e) { - throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + throw new RulesValidationException("Error trying to compile the cleaned rule file: " + mergedRules + " with message: {} " + e.getMessage(), e); } rulesResponse.setRules(mergedRules); return rulesResponse; } + + private enum RuleFileType { + ENTITY, + COMPONENT + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 9e5f180e..65b17db3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -41,7 +41,7 @@ public class RuleBuilderService { } ruleFileBluePrint.dropQueries(); if (removeImports) { - ruleFileBluePrint.dropQueries(); + ruleFileBluePrint.dropImports(); } log.info("Finished removing system rules for ruleFile"); @@ -62,9 +62,7 @@ public class RuleBuilderService { } ruleFileBluePrintExisting.setImports(ruleFileBluePrintUpdate.getImports()); ruleFileBluePrintUpdate.getRuleClasses() - .forEach(ruleClass -> { - ruleFileBluePrintExisting.addRuleClass(ruleClass); - }); + .forEach(ruleFileBluePrintExisting::addRuleClass); return RuleFileParser.buildRulesStringFromBluePrint(ruleFileBluePrintExisting); } -- 2.47.2 From ae38506809357ff0853c37de97e19faeca039fe0 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 7 Oct 2024 13:59:34 +0200 Subject: [PATCH 5/9] RED-9472: seperation of system rules refactored two have a rulefileblueprint and a rulefileblueprint for validationresult changed the flow of merging and seperating system and user rules added functions and declarations to rule file parsing --- .../v1/resources/RuleBuilderResource.java | 3 +- .../controller/RuleBuilderController.java | 55 +++++++++--- .../v1/server/model/RuleMergingResult.java | 24 ++++++ .../v1/server/model/RulesResponse.java | 23 ----- ...ePrint.java => RuleCompilationResult.java} | 2 +- .../v1/server/service/RuleBuilderService.java | 46 +++++----- .../drools/DroolsValidationService.java | 84 +++++++++---------- ....java => RuleCompilationResultParser.java} | 47 ++++++++--- .../services/DroolsValidationServiceTest.java | 16 ++-- .../management/factory/RuleFileFactory.java | 31 ++++--- .../management/factory/RuleFileParser.java | 23 ++++- .../migration/RuleFileMigrator.java | 2 +- .../migration/RuleIdentifierMigrator.java | 12 ++- .../management/models/BasicDeclaration.java | 34 ++++++++ .../management/models/BasicFunction.java | 36 ++++++++ .../management/models/RuleFileBluePrint.java | 40 +++++++-- ... => RuleCompilationResultMergingTest.java} | 2 +- 17 files changed, 338 insertions(+), 142 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RulesResponse.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/{RuleFileBluePrint.java => RuleCompilationResult.java} (99%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/{RuleFileParser.java => RuleCompilationResultParser.java} (79%) create mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicDeclaration.java create mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicFunction.java rename redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/{RuleFileBluePrintMergingTest.java => RuleCompilationResultMergingTest.java} (93%) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index a0711376..4683fd5f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.resources; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -20,6 +21,6 @@ public interface RuleBuilderResource { @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) - RulesResponse getRuleFileWithSeperatedSystemRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); + ResponseEntity mergeUserUpdateRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 09be0f0c..cdfd32d0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,13 +1,20 @@ package com.iqser.red.service.redaction.v1.server.controller; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleBlacklistErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxWarningMessage; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; @@ -36,28 +43,54 @@ public class RuleBuilderController implements RuleBuilderResource { RulesResponse rulesResponse = new RulesResponse(); String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); - try { - droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), filteredRules)); - } catch (Exception e) { - throw new RulesValidationException("Error trying to compile the cleaned rule file: " + filteredRules + " with message: {} " + e.getMessage(), e); - } rulesResponse.setRules(filteredRules); return rulesResponse; } @Override - public RulesResponse getRuleFileWithSeperatedSystemRules(RulesUpdateRequest rulesUpdateRequest) { + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { RulesResponse rulesResponse = new RulesResponse(); - String mergedRules = ruleBuilderService.getRuleFileWithSeperatedSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + DroolsValidationResponse droolsValidationResponse = new DroolsValidationResponse(); + RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); try { - droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergedRules)); + var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + droolsValidationResponse = DroolsValidationResponse.builder() + .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() + .stream() + .map(droolsSyntaxErrorMessage -> new RuleSyntaxErrorMessage(droolsSyntaxErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxErrorMessage.getColumn(), + droolsSyntaxErrorMessage.getMessage())) + .toList()) + .deprecatedWarnings(droolsValidation.getDeprecatedWarnings() + .stream() + .map(droolsSyntaxDeprecatedWarnings -> new RuleSyntaxWarningMessage(droolsSyntaxDeprecatedWarnings.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxDeprecatedWarnings.getColumn(), + droolsSyntaxDeprecatedWarnings.getMessage())) + .toList()) + .blacklistErrorMessages(droolsValidation.getBlacklistErrorMessages() + .stream() + .map(droolsBlacklistErrorMessage -> new RuleBlacklistErrorMessage(droolsBlacklistErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsBlacklistErrorMessage.getColumn(), + droolsBlacklistErrorMessage.getMessage())) + .toList()) + .build(); + if (!droolsValidation.isCompiled()) { + return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + } else { + rulesResponse.setRules(mergingResult.getMergedRules()); + } } catch (Exception e) { - throw new RulesValidationException("Error trying to compile the cleaned rule file: " + mergedRules + " with message: {} " + e.getMessage(), e); + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } - rulesResponse.setRules(mergedRules); - return rulesResponse; + return new ResponseEntity<>(rulesResponse, HttpStatus.OK); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java new file mode 100644 index 00000000..b2cb1b27 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Schema(description = "Object containing a string of Drools rules.") +public class RuleMergingResult { + + @Schema(description = "The merged rules.") + private String mergedRules; + + @Schema(description = "the length of added imports from sytemRules") + private int addedImportsOffset; + @Schema(description = "the length of added globals from sytemRules") + private int addedGlobalsOffset; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RulesResponse.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RulesResponse.java deleted file mode 100644 index 442e249e..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RulesResponse.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.model; - -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@Schema(description = "Object containing a string of Drools rules.") -public class RulesResponse { - - @Schema(description = "The actual string of rules.") - private String rules; - - @Schema(description = "The DossierTemplate Id for these rules") - private String dossierTemplateId; - - @Schema(description = "Bad written rules can lead to timeouts or endless processing. This will be detected by the system and all analyse request for the rules will be rejected. This flag indicates that a timeout was detected and you need to fix the rules") - private boolean timeoutDetected; - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java similarity index 99% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index 8ac3abcb..b69c629c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -26,7 +26,7 @@ import lombok.extern.slf4j.Slf4j; @NoArgsConstructor @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public final class RuleFileBluePrint { +public final class RuleCompilationResult { String imports; int importLine; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 65b17db3..dd5b36f3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -8,9 +8,11 @@ import java.util.List; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; +import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; +import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; +import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; import lombok.extern.slf4j.Slf4j; @@ -37,34 +39,32 @@ public class RuleBuilderService { RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); log.info("Starting to remove system rules from ruleFile"); for (String systemRule : systemRules) { - ruleFileBluePrint.dropRulesByIdentifier(RuleType.fromString(systemRule)); - } - ruleFileBluePrint.dropQueries(); - if (removeImports) { - ruleFileBluePrint.dropImports(); + ruleFileBluePrint.removeRule(RuleIdentifier.fromName(systemRule)); } log.info("Finished removing system rules for ruleFile"); - return RuleFileParser.buildRulesStringFromBluePrint(ruleFileBluePrint); + // removing imports if flagged and dropping queries when building result string + return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); } - public String getRuleFileWithSeperatedSystemRules(String existingRules, String updatedRules) { - + public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String updatedRules) { + //todo: when returning validation result to frotnend substract the length of imports/globals + log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - ruleFileBluePrintExisting.dropAllRulesExceptSystemRules(systemRules); - RuleFileBluePrint ruleFileBluePrintUpdate = RuleFileParser.buildBluePrintFromRulesString(updatedRules); - for (String systemRule : systemRules) { - if (ruleFileBluePrintUpdate.countRuleOccurences(RuleType.fromString(systemRule)) > 0) { - throw new RuntimeException("System rules are not allowed in an update."); + RuleFileBluePrint ruleFileBluePrintExistingMerged = RuleFileParser.buildBluePrintFromRulesString(updatedRules); + // add imports from systemrules + ruleFileBluePrintExistingMerged.setImports(ruleFileBluePrintExisting.getImports() + ruleFileBluePrintExistingMerged.getImports()); + //add globals from systemrules + ruleFileBluePrintExistingMerged.setGlobals(ruleFileBluePrintExisting.getGlobals() + ruleFileBluePrintExistingMerged.getGlobals()); + log.info("finished merging user rules update with system rules"); + RuleMergingResult mergingResult = RuleMergingResult.builder() + .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) + .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) + .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .build(); - } - } - ruleFileBluePrintExisting.setImports(ruleFileBluePrintUpdate.getImports()); - ruleFileBluePrintUpdate.getRuleClasses() - .forEach(ruleFileBluePrintExisting::addRuleClass); - - return RuleFileParser.buildRulesStringFromBluePrint(ruleFileBluePrintExisting); + return mergingResult; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index 6d98ae0c..df73f4b8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -28,7 +28,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplemen import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; @@ -69,70 +69,70 @@ public class DroolsValidationService { private DroolsValidation buildCustomDroolsValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation(); + DroolsValidation customValidation = ruleCompilationResult.getDroolsValidation(); - addSyntaxDeprecatedWarnings(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxDeprecatedWarnings(ruleFileType, ruleCompilationResult, customValidation); - addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxErrorMessages(ruleFileType, ruleCompilationResult, customValidation); if (redactionServiceSettings.isRuleExecutionSecured()) { - addBlacklistErrorMessages(ruleFileBluePrint, customValidation); + addBlacklistErrorMessages(ruleCompilationResult, customValidation); } return customValidation; } - private void addSyntaxDeprecatedWarnings(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxDeprecatedWarnings(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { // find deprecated elements in the ruleFileBluePrint - DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); + DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleCompilationResult); if (warningMessageForImports != null) { customValidation.getDeprecatedWarnings().add(warningMessageForImports); } - customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); + customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleCompilationResult)); if (ruleFileType.equals(RuleFileType.COMPONENT)) { - if (!ruleFileBluePrint.getGlobals().contains(ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL)) { - customValidation.getDeprecatedWarnings().add(buildComponentMappingServiceMissingMessage(ruleFileBluePrint)); + if (!ruleCompilationResult.getGlobals().contains(ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL)) { + customValidation.getDeprecatedWarnings().add(buildComponentMappingServiceMissingMessage(ruleCompilationResult)); } } } - private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleFileBluePrint ruleFileBluePrint) { + private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleCompilationResult ruleCompilationResult) { return DroolsSyntaxDeprecatedWarnings.builder() .message("global ComponentMappingService " + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL + "\n is missing from the rules, consider adding it, as it will be required in future versions!") - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) .build(); } - private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { + private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleCompilationResult ruleCompilationResult) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { - String imports = ruleFileBluePrint.getImports(); + String imports = ruleCompilationResult.getImports(); SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation(); List matches = classesSearchImplementation.getMatches(imports); if (!matches.isEmpty()) { String sb = "Following imports are deprecated: \n" + matches.stream() .map(m -> imports.substring(m.startIndex(), m.endIndex())) .collect(Collectors.joining("\n")); - return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build(); + return DroolsSyntaxDeprecatedWarnings.builder().line(ruleCompilationResult.getImportLine()).column(0).message(sb).build(); } } return null; } - private List getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { + private List getWarningsForDeprecatedRules(RuleCompilationResult ruleCompilationResult) { List warningMessages = new ArrayList<>(); @@ -141,7 +141,7 @@ public class DroolsValidationService { SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation(); Map deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap(); - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { List matches = methodsSearchImplementation.getMatches(basicRule.getCode()); @@ -165,32 +165,32 @@ public class DroolsValidationService { } - private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { - RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { - case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); - case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + RuleCompilationResult baseRuleCompilationResult = switch (ruleFileType) { + case ENTITY -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); + case COMPONENT -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseComponentRuleFileString()); }; - if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) { + if (!importsAreValid(baseRuleCompilationResult, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getImportLine()) + .line(ruleCompilationResult.getImportLine()) .column(0) - .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports())) + .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleCompilationResult.getImports())) .build()); } - if (!ruleFileBluePrint.getGlobals().contains(baseRuleFileBluePrint.getGlobals())) { + if (!ruleCompilationResult.getGlobals().contains(baseRuleCompilationResult.getGlobals())) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) - .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals())) + .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleCompilationResult.getGlobals())) .build()); } - baseRuleFileBluePrint.getQueries() + baseRuleCompilationResult.getQueries() .forEach(basicQuery -> { - if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { + if (!validateQueryIsPresent(basicQuery, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(basicQuery.getLine()) @@ -201,7 +201,7 @@ public class DroolsValidationService { }); if (ruleFileType.equals(RuleFileType.ENTITY)) { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; - if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { + if (!validateAgendaGroupIsPresent(ruleCompilationResult, requiredAgendaGroup)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(0) @@ -213,18 +213,18 @@ public class DroolsValidationService { } - private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) { + private boolean validateAgendaGroupIsPresent(RuleCompilationResult ruleCompilationResult, String agendaGroupName) { - return ruleFileBluePrint.streamAllRules() + return ruleCompilationResult.streamAllRules() .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName)); } - private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) { + private boolean importsAreValid(RuleCompilationResult baseRuleCompilationResult, RuleCompilationResult ruleCompilationResult) { // imports may shrink, but not add anything new! - Set baseImports = baseRuleFileBluePrint.getImportSplitByKeyword(); - Set imports = ruleFileBluePrint.getImportSplitByKeyword(); + Set baseImports = baseRuleCompilationResult.getImportSplitByKeyword(); + Set imports = ruleCompilationResult.getImportSplitByKeyword(); Set additionalImports = Sets.difference(imports, baseImports); @@ -233,15 +233,15 @@ public class DroolsValidationService { } - private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) { + private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleCompilationResult ruleCompilationResult) { - return ruleFileBluePrint.getQueries() + return ruleCompilationResult.getQueries() .stream() .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode())); } - private void addBlacklistErrorMessages(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addBlacklistErrorMessages(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { List blacklistErrorMessages = new ArrayList<>(); @@ -253,14 +253,14 @@ public class DroolsValidationService { // check also the imports DroolsBlacklistErrorMessage blacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, - ruleFileBluePrint.getImports(), - ruleFileBluePrint.getImportLine()); + ruleCompilationResult.getImports(), + ruleCompilationResult.getImportLine()); if (blacklistErrorMessage != null) { blacklistErrorMessages.add(blacklistErrorMessage); } // check the rules - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { DroolsBlacklistErrorMessage ruleBlacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java similarity index 79% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index efd2976c..1d6561e9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -20,7 +20,7 @@ import com.iqser.red.service.redaction.v1.model.DroolsValidation; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; @@ -29,14 +29,38 @@ import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @UtilityClass -public class RuleFileParser { +public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); + List allRules = new LinkedList<>(); + List allQueries = new LinkedList<>(); + PackageDescr packageDescr = new PackageDescr(); + String imports = ""; + String globals =""; + List ruleClasses = buildRuleClasses(allRules); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } @SneakyThrows - public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); @@ -65,26 +89,26 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), - packageDescr.getImports() + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() .stream() .findFirst() .map(ImportDescr::getLine) .orElse(0), - globals.trim(), - packageDescr.getGlobals() + globals.trim(), + packageDescr.getGlobals() .stream() .findFirst() .map(GlobalDescr::getLine) .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); + allQueries, + ruleClasses, + customDroolsValidation); } @SneakyThrows - public String buildRulesStringFromBluePrint(RuleFileBluePrint bluePrint) { + public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { StringBuilder ruleStringBuilder = new StringBuilder(); @@ -179,6 +203,7 @@ public class RuleFileParser { private List buildRuleClasses(List allRules) { + //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index b8fd95fe..2e42db62 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -22,11 +22,11 @@ import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; import lombok.SneakyThrows; @@ -235,11 +235,11 @@ class DroolsValidationServiceTest { if (droolsValidation.isCompiled()) { continue; } - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(rulesString); + RuleCompilationResult baseRuleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); - rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); - rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + rulesString = rulesString.replace(ruleCompilationResult.getImports(), baseRuleCompilationResult.getImports()); + rulesString = rulesString.replace(ruleCompilationResult.getGlobals(), baseRuleCompilationResult.getGlobals()); try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); @@ -460,9 +460,9 @@ class DroolsValidationServiceTest { end """; - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled()); + assertFalse(ruleCompilationResult.getDroolsValidation().isCompiled()); } } \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java index 039664bf..8bc7fd73 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java @@ -48,17 +48,17 @@ public class RuleFileFactory { } RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers); - return buildRuleString(filteredBluePrint); + return buildRuleString(filteredBluePrint, false, false); } @SneakyThrows - public String buildRuleString(RuleFileBluePrint bluePrint) { + public String buildRuleString(RuleFileBluePrint bluePrint, boolean dropImports, boolean dropQueries) { try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) { String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8); List templateRuleOrder = parseRuleOrder(template); - return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder); + return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder, dropImports, dropQueries); } } @@ -72,23 +72,31 @@ public class RuleFileFactory { } - private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder) { + private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder, boolean dropImports, boolean dropQueries) { - Set additionalRuleTypes = bluePrint.ruleClasses() + Set additionalRuleTypes = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleType) .filter(ruleType -> !ruleOrder.contains(ruleType)) .collect(Collectors.toSet()); StringBuilder sb = new StringBuilder(); - sb.append(bluePrint.imports()); + if (!dropImports) { + sb.append(bluePrint.getImports()); + sb.append("\n\n"); + } + sb.append(bluePrint.getGlobals()); sb.append("\n\n"); - sb.append(bluePrint.globals()); + sb.append("//------------------------------------ declarations ------------------------------------"); sb.append("\n\n"); - sb.append("//------------------------------------ queries ------------------------------------"); - sb.append("\n\n"); - sb.append(bluePrint.queries()); + sb.append(bluePrint.getDeclarations()); sb.append("\n\n"); + if (!dropQueries) { + sb.append("//------------------------------------ queries ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getQueries()); + sb.append("\n\n"); + } for (RuleType ruleBlockType : ruleOrder) { if (ruleBlockType.isWildCard()) { additionalRuleTypes.stream() @@ -101,6 +109,9 @@ public class RuleFileFactory { } writeRuleClass(bluePrint, ruleBlockType, sb); } + sb.append("//------------------------------------ functions ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getFunctions()); return sb.toString().trim() + "\n"; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java index 3e17b0db..15a9b05f 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java @@ -10,6 +10,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; +import org.drools.drl.ast.descr.FunctionDescr; import org.drools.drl.ast.descr.ImportDescr; import org.drools.drl.ast.descr.PackageDescr; import org.drools.drl.ast.descr.RuleDescr; @@ -18,6 +20,8 @@ import org.kie.internal.builder.conf.LanguageLevelOption; import com.knecon.fforesight.utility.rules.management.RuleManagementResources; import com.knecon.fforesight.utility.rules.management.models.ApplicationType; +import com.knecon.fforesight.utility.rules.management.models.BasicDeclaration; +import com.knecon.fforesight.utility.rules.management.models.BasicFunction; import com.knecon.fforesight.utility.rules.management.models.BasicRule; import com.knecon.fforesight.utility.rules.management.models.RuleClass; import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; @@ -49,6 +53,8 @@ public class RuleFileParser { PackageDescr packageDescr = parser.parse(false, rulesString); StringBuilder queryBuilder = new StringBuilder(); List allRules = new LinkedList<>(); + List functions = new LinkedList<>(); + List declarations = new LinkedList<>(); for (RuleDescr rule : packageDescr.getRules()) { if (rule.isQuery()) { queryBuilder.append(rulesString, rule.getStartCharacter(), rule.getEndCharacter()); @@ -57,6 +63,21 @@ public class RuleFileParser { } allRules.add(BasicRule.fromRuleDescr(rule, rulesString)); } + for (FunctionDescr function : packageDescr.getFunctions()) { + functions.add(BasicFunction.fromFunctionDescr(function, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getEnumDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getClassAndEnumDeclarationDescrs()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } String imports = rulesString.substring(0, packageDescr.getImports() .stream() @@ -70,7 +91,7 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses); + return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses, declarations, functions); } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java index 94975b8e..7a031490 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java @@ -33,7 +33,7 @@ public class RuleFileMigrator { //replaceRules(ruleFileBluePrint, combinedBluePrint); replaceRuleIdentifiers(combinedBluePrint, ruleFileBluePrint); - String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); + String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint,false, false); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java index fc8e9026..20a0ab5f 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java @@ -46,7 +46,7 @@ public class RuleIdentifierMigrator { bluePrint = migrateMatchedRuleForAllRules(bluePrint); - String ruleString = RuleFileFactory.buildRuleString(bluePrint); + String ruleString = RuleFileFactory.buildRuleString(bluePrint, false, false); try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) { out.write(ruleString.getBytes(StandardCharsets.UTF_8)); } @@ -57,9 +57,10 @@ public class RuleIdentifierMigrator { } + //todo: introduce functions and declarations private static RuleFileBluePrint migrateMatchedRuleForAllRules(RuleFileBluePrint bluePrint) { - List migratedRules = bluePrint.ruleClasses() + List migratedRules = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) @@ -67,7 +68,12 @@ public class RuleIdentifierMigrator { .flatMap(Collection::stream) .map(RuleIdentifierMigrator::migrateMatchedRule) .toList(); - RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.imports(), bluePrint.globals(), bluePrint.queries(), new LinkedList<>()); + RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.getImports(), + bluePrint.getGlobals(), + bluePrint.getQueries(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>()); migratedRules.forEach(migratedBluePrint::addRule); return migratedBluePrint; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicDeclaration.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicDeclaration.java new file mode 100644 index 00000000..5798601f --- /dev/null +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicDeclaration.java @@ -0,0 +1,34 @@ +package com.knecon.fforesight.utility.rules.management.models; + +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicDeclaration { + + @EqualsAndHashCode.Include + String declarationFullTypeName; + String body; + int line; + + + public static BasicDeclaration fromDeclarationDescription(AbstractClassTypeDeclarationDescr declaration, String rulesString) { + + String declarationFullTypeName = declaration.getFullTypeName(); + String body = rulesString.substring(declaration.getStartCharacter(), declaration.getEndCharacter()); + int line = declaration.getLine(); + + return new BasicDeclaration(declarationFullTypeName, body, line); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicFunction.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicFunction.java new file mode 100644 index 00000000..c707ba79 --- /dev/null +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicFunction.java @@ -0,0 +1,36 @@ +package com.knecon.fforesight.utility.rules.management.models; + +import org.drools.drl.ast.descr.FunctionDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicFunction { + + @EqualsAndHashCode.Include + String functionName; + String returnType; + String body; + int line; + + + public static BasicFunction fromFunctionDescr(FunctionDescr function, String rulesString) { + + String functionName = function.getName(); + String returnType = function.getReturnType(); + String body = rulesString.substring(function.getStartCharacter(), function.getEndCharacter()); + int line = function.getLine(); + + return new BasicFunction(functionName, returnType, body, line); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java index 2b2ae91f..50242175 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java @@ -10,7 +10,29 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -public record RuleFileBluePrint(String imports, String globals, String queries, List ruleClasses) { +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RuleFileBluePrint { + + String imports; + String globals; + String queries; + List ruleClasses; + List declarations; + List functions; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -24,7 +46,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, ruleClass.ruleUnits().remove(ruleUnit); } if (ruleClass.ruleUnits().isEmpty()) { - ruleClasses().remove(ruleClass); + this.ruleClasses.remove(ruleClass); } })); return wasRemoved.get(); @@ -41,7 +63,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRules() { - return ruleClasses().stream() + return this.ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -101,7 +123,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRuleIdentifiers() { - return ruleClasses().stream() + return ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -122,10 +144,16 @@ public record RuleFileBluePrint(String imports, String globals, String queries, } + public void setImports(String newImports) { + + this.imports = newImports; + } + + public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>()); - ruleClasses().stream() + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) .flatMap(ruleUnit -> ruleUnit.rules() diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleCompilationResultMergingTest.java similarity index 93% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java rename to redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleCompilationResultMergingTest.java index 559101ac..f9fe18f4 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleCompilationResultMergingTest.java @@ -8,7 +8,7 @@ import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -public class RuleFileBluePrintMergingTest { +public class RuleCompilationResultMergingTest { @Test public void testBothRuleFilesCanBeMerged() { -- 2.47.2 From a8df56b2009e5ffb9b2f56b57a706d12b9a12a90 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 7 Oct 2024 14:31:14 +0200 Subject: [PATCH 6/9] RED-9472: seperation of system rules --- .../v1/server/service/RuleBuilderService.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index dd5b36f3..45947272 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; @@ -36,6 +37,8 @@ public class RuleBuilderService { public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { + removeImports = true; + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); log.info("Starting to remove system rules from ruleFile"); for (String systemRule : systemRules) { @@ -49,10 +52,23 @@ public class RuleBuilderService { public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String updatedRules) { - //todo: when returning validation result to frotnend substract the length of imports/globals + log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); RuleFileBluePrint ruleFileBluePrintExistingMerged = RuleFileParser.buildBluePrintFromRulesString(updatedRules); + ruleFileBluePrintExisting.getAllRuleIdentifiers() + .stream() + .filter(ruleIdentifier -> { + for (String systemRule : systemRules) { + return !ruleIdentifier.matches(RuleIdentifier.fromString(systemRule)); + } + return false; + }) + .collect(Collectors.toList()) + .forEach(rule -> { + log.info("removing rule: {}",rule); + ruleFileBluePrintExisting.removeRule(rule); + }); // add imports from systemrules ruleFileBluePrintExistingMerged.setImports(ruleFileBluePrintExisting.getImports() + ruleFileBluePrintExistingMerged.getImports()); //add globals from systemrules -- 2.47.2 From 3f043c5fdda488afd0d9a131dcf5cd77f09b1049 Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 8 Oct 2024 17:32:33 +0200 Subject: [PATCH 7/9] RED-9472: seperation of system rules introduced tests working on fixing bug with merging rules --- .../controller/RuleBuilderController.java | 5 +- .../v1/server/service/RuleBuilderService.java | 105 +++++++++---- .../drools/RuleCompilationResultParser.java | 61 +++++--- .../redaction/v1/server/RuleBuilderTest.java | 141 ++++++++++++++++++ .../resources/drools/user_rule_update.drl | 49 ++++++ .../management/factory/RuleFileFactory.java | 8 +- 6 files changed, 317 insertions(+), 52 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/user_rule_update.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index cdfd32d0..23e5c9d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.controller; +import org.junit.Assert; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; @@ -43,6 +44,7 @@ public class RuleBuilderController implements RuleBuilderResource { RulesResponse rulesResponse = new RulesResponse(); String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); + Assert.assertFalse("There was an error when cleaning the rulefile of system rules", filteredRules.isEmpty()); rulesResponse.setRules(filteredRules); return rulesResponse; } @@ -52,8 +54,9 @@ public class RuleBuilderController implements RuleBuilderResource { public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { RulesResponse rulesResponse = new RulesResponse(); - DroolsValidationResponse droolsValidationResponse = new DroolsValidationResponse(); + DroolsValidationResponse droolsValidationResponse; RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + Assert.assertFalse("There was an error when merging the user rule update", mergingResult.getMergedRules().isEmpty()); try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); droolsValidationResponse = DroolsValidationResponse.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 45947272..0fd4d255 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -4,8 +4,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; +import org.junit.Assert; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; @@ -14,6 +16,8 @@ import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; +import com.knecon.fforesight.utility.rules.management.models.RuleType; +import com.knecon.fforesight.utility.rules.management.models.RuleUnit; import lombok.extern.slf4j.Slf4j; @@ -21,7 +25,13 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + //todo: make this configurable + private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), + new RuleType("MAN"), + new RuleType("X"), + new RuleType("DICT"), + new RuleType("FA"), + new RuleType("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -37,42 +47,30 @@ public class RuleBuilderService { public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { - removeImports = true; - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); log.info("Starting to remove system rules from ruleFile"); - for (String systemRule : systemRules) { - ruleFileBluePrint.removeRule(RuleIdentifier.fromName(systemRule)); - } + removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); log.info("Finished removing system rules for ruleFile"); - // removing imports if flagged and dropping queries when building result string return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); } - public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String updatedRules) { + public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint ruleFileBluePrintExistingMerged = RuleFileParser.buildBluePrintFromRulesString(updatedRules); - ruleFileBluePrintExisting.getAllRuleIdentifiers() + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules); + removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getRuleClasses() .stream() - .filter(ruleIdentifier -> { - for (String systemRule : systemRules) { - return !ruleIdentifier.matches(RuleIdentifier.fromString(systemRule)); - } - return false; - }) - .collect(Collectors.toList()) - .forEach(rule -> { - log.info("removing rule: {}",rule); - ruleFileBluePrintExisting.removeRule(rule); - }); - // add imports from systemrules - ruleFileBluePrintExistingMerged.setImports(ruleFileBluePrintExisting.getImports() + ruleFileBluePrintExistingMerged.getImports()); - //add globals from systemrules - ruleFileBluePrintExistingMerged.setGlobals(ruleFileBluePrintExisting.getGlobals() + ruleFileBluePrintExistingMerged.getGlobals()); + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .flatMap(ruleUnit -> ruleUnit.rules() + .stream())) + .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); + mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); log.info("finished merging user rules update with system rules"); RuleMergingResult mergingResult = RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) @@ -83,4 +81,61 @@ public class RuleBuilderService { return mergingResult; } + + private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { + + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + removeAllRulesExceptSystemRules(ruleFileBluePrint); + ruleFileBluePrint.getRuleClasses() + .forEach(ruleClass -> Assert.assertFalse("there was an error removing all rules except system rules", systemRuleNames.contains(ruleClass.ruleType().name()))); + } + + + private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + + ruleFileBluePrint.getRuleClasses() + .stream() + .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) + .forEach(ruleFileBluePrint::removeRule); + } + + + private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { + + removeSystemRules(ruleFileBluePrint); + for (RuleType systemRule : systemRules) { + List remainingSystemRules = new ArrayList(); + ruleFileBluePrint.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(remainingSystemRules::add)); + Assert.assertTrue("There was an error removing the system rules from the file", remainingSystemRules.isEmpty()); + } + } + + + private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { + + for (RuleType systemRule : systemRules) { + List rules = new ArrayList(); + ruleFileBluePrintExisting.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(rules::add)); + rules.forEach(ruleUnit -> { + ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); + }); + ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 1d6561e9..2a5158f3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -34,33 +34,36 @@ public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); List allRules = new LinkedList<>(); List allQueries = new LinkedList<>(); PackageDescr packageDescr = new PackageDescr(); String imports = ""; - String globals =""; + String globals = ""; List ruleClasses = buildRuleClasses(allRules); return new RuleCompilationResult(imports.trim(), packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), globals.trim(), packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), allQueries, ruleClasses, customDroolsValidation); } + @SneakyThrows - public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); @@ -76,12 +79,15 @@ public class RuleCompilationResultParser { } } - String imports = ruleString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + String imports = ""; + if (!removedImports) { + imports = ruleString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -91,22 +97,29 @@ public class RuleCompilationResultParser { return new RuleCompilationResult(imports.trim(), packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), globals.trim(), packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), allQueries, ruleClasses, customDroolsValidation); } + @SneakyThrows + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { + + return buildRuleCompilationResultFromRuleString(ruleString,false); + } + + @SneakyThrows public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java new file mode 100644 index 00000000..9dcbeb4a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -0,0 +1,141 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RuleBuilderTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); + private static final String USER_RULES = loadFromClassPath("drools/user_rule_update.drl"); + private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + + @Autowired + RuleBuilderService ruleBuilderService; + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import(LayoutParsingServiceProcessorConfiguration.class) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RuleBuilderTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + public void removeSystemRulesTest() { + + String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); + String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + + Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + + Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + } + + + @Test + public void mergeUserRulesUpdateTest() { + + RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); + String mergedRules = mergingResult.getMergedRules(); + + } + + + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getRuleClasses() + .stream() + .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) + .collect(Collectors.toList()).size() == 0; + } + + + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getImports().isEmpty(); + } + + + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getQueries().isEmpty(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/user_rule_update.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/user_rule_update.drl new file mode 100644 index 00000000..a4828b80 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/user_rule_update.drl @@ -0,0 +1,49 @@ +//------------------------------------ Table extraction rules ------------------------------------ + +// Rule unit: TAB.0 +rule "TAB.0.0: Changed Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Changed Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +// Rule unit: TAB.6 +rule "TAB.6.0: Changed Targeted cell extraction (Experimental Stop date)" + when + $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2")) + $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList() + TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList() + $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList() + then + entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date")); + end + diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java index 8bc7fd73..31d653d1 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java @@ -84,12 +84,16 @@ public class RuleFileFactory { if (!dropImports) { sb.append(bluePrint.getImports()); sb.append("\n\n"); + } else { + //todo: this is hacked to enable compiling the rules without imports + sb.append("package drools"); + sb.append("\n\n"); } sb.append(bluePrint.getGlobals()); sb.append("\n\n"); sb.append("//------------------------------------ declarations ------------------------------------"); sb.append("\n\n"); - sb.append(bluePrint.getDeclarations()); + sb.append(bluePrint.getDeclarations().isEmpty() ? "" : bluePrint.getDeclarations()); sb.append("\n\n"); if (!dropQueries) { sb.append("//------------------------------------ queries ------------------------------------"); @@ -111,7 +115,7 @@ public class RuleFileFactory { } sb.append("//------------------------------------ functions ------------------------------------"); sb.append("\n\n"); - sb.append(bluePrint.getFunctions()); + sb.append(bluePrint.getFunctions().isEmpty() ? "" : bluePrint.getFunctions()); return sb.toString().trim() + "\n"; } -- 2.47.2 From 56011341b7ac2939884015821d19730f1540ca84 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 9 Oct 2024 12:17:49 +0200 Subject: [PATCH 8/9] RED-9472: seperation of system rules introduced tests --- .../build.gradle.kts | 1 + .../v1/server/service/RuleBuilderService.java | 12 ++++-- .../redaction/v1/server/RuleBuilderTest.java | 8 ++++ .../resources/drools/user_rule_update.drl | 3 +- .../management/factory/RuleFileParser.java | 41 +++++++++++++------ 5 files changed, 47 insertions(+), 18 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index a7061434..44de1f8d 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -33,6 +33,7 @@ configurations { dependencies { + implementation(project(":rules-management")) implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 0fd4d255..95cecf24 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -60,8 +60,8 @@ public class RuleBuilderService { log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules); - removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); ruleFileBluePrintExisting.getRuleClasses() .stream() .flatMap(ruleClass -> ruleClass.ruleUnits() @@ -89,12 +89,13 @@ public class RuleBuilderService { .collect(Collectors.toSet()); removeAllRulesExceptSystemRules(ruleFileBluePrint); ruleFileBluePrint.getRuleClasses() - .forEach(ruleClass -> Assert.assertFalse("there was an error removing all rules except system rules", systemRuleNames.contains(ruleClass.ruleType().name()))); + .forEach(ruleClass -> Assert.assertTrue("there was an error removing all rules except system rules", systemRuleNames.contains(ruleClass.ruleType().name()))); } private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + List rules = new ArrayList(); Set systemRuleNames = systemRules.stream() .map(RuleType::name) .collect(Collectors.toSet()); @@ -105,7 +106,10 @@ public class RuleBuilderService { .flatMap(ruleClass -> ruleClass.ruleUnits() .stream() .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) - .forEach(ruleFileBluePrint::removeRule); + .forEach(rule -> rules.add(rule)); + rules.forEach(ruleIdentifier -> { + ruleFileBluePrint.removeRule(ruleIdentifier); + }); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 9dcbeb4a..de4592ff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -37,6 +37,8 @@ import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; +import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; +import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -114,6 +116,12 @@ public class RuleBuilderTest extends AbstractRedactionIntegrationTest { RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); String mergedRules = mergingResult.getMergedRules(); + RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); + ruleFileBluePrintUserRulesUpdate.getRuleClasses() + .forEach(ruleClass -> { + Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); + }); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/user_rule_update.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/user_rule_update.drl index a4828b80..16ea73d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/user_rule_update.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/user_rule_update.drl @@ -1,3 +1,4 @@ +package drools //------------------------------------ Table extraction rules ------------------------------------ // Rule unit: TAB.0 @@ -35,7 +36,7 @@ rule "TAB.0.1: Changed Guidelines" // Rule unit: TAB.6 -rule "TAB.6.0: Changed Targeted cell extraction (Experimental Stop date)" +rule "TAB.6.1: Changed Targeted cell extraction (Experimental Stop date)" when $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2")) $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList() diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java index 15a9b05f..36b206aa 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java @@ -49,6 +49,13 @@ public class RuleFileParser { @SneakyThrows public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString) { + return buildBluePrintFromRulesString(rulesString, false); + } + + + @SneakyThrows + public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString, boolean isUserRuleUpdate) { + DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); PackageDescr packageDescr = parser.parse(false, rulesString); StringBuilder queryBuilder = new StringBuilder(); @@ -78,12 +85,15 @@ public class RuleFileParser { for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); } - String imports = rulesString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + String imports = ""; + if (isUserRuleUpdate && !packageDescr.getImports().isEmpty()) { + imports = rulesString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -108,13 +118,18 @@ public class RuleFileParser { private List groupingByGroup(List rules) { - Map> rulesPerUnit = rules.stream() - .collect(groupingBy(rule -> rule.identifier().unit())); - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); + try { + Map> rulesPerUnit = rules.stream() + .collect(groupingBy(rule -> rule.identifier().unit())); + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); + } catch (NullPointerException e) { + System.out.println(rules); + throw e; + } } -- 2.47.2 From 9c3dc2653b2d250886d8556745e73582a68d82fe Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 9 Oct 2024 12:21:31 +0200 Subject: [PATCH 9/9] RED-9472: seperation of system rules fixed circular depdency --- redaction-service-v1/rules-management/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/redaction-service-v1/rules-management/build.gradle.kts b/redaction-service-v1/rules-management/build.gradle.kts index 2aaea40c..b3ef9cfa 100644 --- a/redaction-service-v1/rules-management/build.gradle.kts +++ b/redaction-service-v1/rules-management/build.gradle.kts @@ -27,7 +27,6 @@ sourceSets { } dependencies { - implementation(project(":redaction-service-server-v1")) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") -- 2.47.2