diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 87ec3822..dc942464 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,11 +4,12 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.631.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" dependencies { implementation("org.springframework:spring-web:6.0.12") implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + api("com.knecon.fforesight:swagger-commons:0.7.0") } publishing { diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index 3864ba9b..4683fd5f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,13 +1,26 @@ package com.iqser.red.service.redaction.v1.resources; -import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; - import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; public interface RuleBuilderResource { @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE) RuleBuilderModel getRuleBuilderModel(); + + @PostMapping(value = "/internal-api/rules/user-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithoutSystemRules(@RequestBody SystemRulesSeperationRequest systemRulesSeperationRequest); + + + @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + ResponseEntity mergeUserUpdateRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); + } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 262655fd..f3d723ae 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.641.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" @@ -31,10 +31,15 @@ configurations { } } +configurations.all { + resolutionStrategy { + force("com.google.protobuf:protobuf-java:4.27.1") + } +} + dependencies { implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } - implementation(project(":document")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { @@ -47,7 +52,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.50.0") - implementation("com.knecon.fforesight:tenant-commons:0.31.0") + implementation("com.knecon.fforesight:tenant-commons:0.30.0") implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } @@ -56,14 +61,13 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") - implementation("org.ahocorasick:ahocorasick:0.9.0") - implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") - implementation("com.github.roklenarcic:aho-corasick:1.2") + implementation("org.ahocorasick:ahocorasick:0.6.3") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") implementation("org.drools:drools-mvel:${droolsVersion}") implementation("org.kie:kie-spring:7.74.1.Final") + implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.locationtech.jts:jts-core:1.19.0") @@ -82,6 +86,7 @@ dependencies { implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") + api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") @@ -132,7 +137,6 @@ tasks.named("bootBuildImage") { "BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5" ) - environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE! if (project.hasProperty("buildbootDockerHostNetwork")) { @@ -187,19 +191,13 @@ tasks.register("generateJavaDoc", Javadoc::class) { dependsOn("compileJava") dependsOn("delombok") classpath = project.sourceSets["main"].runtimeClasspath - val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { + source = fileTree("${buildDir}/generated/sources/delombok/java/main") { include(droolsImports) } - val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { - include(droolsImports) - } - source = documentFiles + mainFiles - - setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "")) + destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "") options.memberLevel = JavadocMemberLevel.PUBLIC (options as StandardJavadocDocletOptions).apply { title = "API Documentation for Redaction Service ${project.version}" } } - diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 306dccbe..8f3a51dd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,27 +1,111 @@ package com.iqser.red.service.redaction.v1.server.controller; -import java.util.Collections; - +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleBlacklistErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxWarningMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @RestController +@Slf4j @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { + private final RuleBuilderService ruleBuilderService; + private final DroolsValidationService droolsValidationService; + + @Override public RuleBuilderModel getRuleBuilderModel() { - RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + return this.ruleBuilderService.getRuleBuilderModel(); + } - ruleBuilderModel.setWhenClauses(Collections.emptyList()); - ruleBuilderModel.setThenConditions(Collections.emptyList()); - return ruleBuilderModel; + @Override + public RulesResponse getRuleFileWithoutSystemRules(SystemRulesSeperationRequest systemRulesSeperationRequest) { + + RulesResponse rulesResponse = new RulesResponse(); + String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); + if (filteredRules.isEmpty()) { + throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + } + rulesResponse.setRules(filteredRules); + return rulesResponse; + } + + + @Override + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + + RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); + DroolsValidationResponse droolsValidationResponse; + RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + if (mergingResult.getMergedRules().isEmpty()) { + throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + } + try { + var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + droolsValidationResponse = DroolsValidationResponse.builder() + .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() + .stream() + .map(droolsSyntaxErrorMessage -> new RuleSyntaxErrorMessage(droolsSyntaxErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxErrorMessage.getColumn(), + droolsSyntaxErrorMessage.getMessage())) + .toList()) + .deprecatedWarnings(droolsValidation.getDeprecatedWarnings() + .stream() + .map(droolsSyntaxDeprecatedWarnings -> new RuleSyntaxWarningMessage(droolsSyntaxDeprecatedWarnings.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxDeprecatedWarnings.getColumn(), + droolsSyntaxDeprecatedWarnings.getMessage())) + .toList()) + .blacklistErrorMessages(droolsValidation.getBlacklistErrorMessages() + .stream() + .map(droolsBlacklistErrorMessage -> new RuleBlacklistErrorMessage(droolsBlacklistErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsBlacklistErrorMessage.getColumn(), + droolsBlacklistErrorMessage.getMessage())) + .toList()) + .build(); + if (!droolsValidation.isCompiled()) { + return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + } else { + // + rulesUploadResponse.setRules(mergingResult.getMergedRules()); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + } + } catch (Exception e) { + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + } + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); + } + + + private enum RuleFileType { + ENTITY, + COMPONENT } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java new file mode 100644 index 00000000..b2cb1b27 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Schema(description = "Object containing a string of Drools rules.") +public class RuleMergingResult { + + @Schema(description = "The merged rules.") + private String mergedRules; + + @Schema(description = "the length of added imports from sytemRules") + private int addedImportsOffset; + @Schema(description = "the length of added globals from sytemRules") + private int addedGlobalsOffset; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java similarity index 61% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index 3aa1cdef..e96aa697 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -20,13 +20,15 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Builder @NoArgsConstructor @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public final class RuleFileBluePrint { +public final class RuleCompilationResult { String imports; int importLine; @@ -45,9 +47,62 @@ public final class RuleFileBluePrint { } + public void dropRulesByIdentifier(RuleType ruleType) { + + log.info("removing rule with identifier {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropAllRulesExceptSystemRules(List systemRules) { + + log.info("removing all rules except the system rules"); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropImports() { + + this.imports = ""; + } + + + public void dropQueries() { + + this.queries.clear(); + } + + + public int countRuleOccurences(RuleType ruleType) { + + log.info("counting occurences of files {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + return rulesToBeRemoved.size(); + } + + + private boolean filterOutRule(RuleType ruleType, List filteredRules) { + + return !filteredRules.contains(ruleType.name()); + } + + public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replaceAll("\n", "").split("import")) + return Arrays.stream(imports.replace("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } @@ -76,7 +131,7 @@ public final class RuleFileBluePrint { public List getAllRuleIdentifiers() { return streamAllRules().map(BasicRule::getIdentifier) - .collect(Collectors.toList()); + .toList(); } @@ -96,4 +151,10 @@ public final class RuleFileBluePrint { return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']'; } + + public void addRuleClass(RuleClass ruleClass) { + + this.ruleClasses.add(ruleClass); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java new file mode 100644 index 00000000..331b1e25 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import java.io.InputStream; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; + +@SuppressWarnings("PMD") +public class RuleManagementResources { + + public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + } + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + } + + + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + } else { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + } + } + + + public static InputStream getTemplateInputStream() { + + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java index 2a8de641..c83f7f04 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java @@ -1,12 +1,12 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java index 039664bf..f6332332 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.Collections.emptySet; @@ -9,13 +9,13 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -48,17 +48,17 @@ public class RuleFileFactory { } RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers); - return buildRuleString(filteredBluePrint); + return buildRuleString(filteredBluePrint, false, false); } @SneakyThrows - public String buildRuleString(RuleFileBluePrint bluePrint) { + public String buildRuleString(RuleFileBluePrint bluePrint, boolean dropImports, boolean dropQueries) { try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) { String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8); List templateRuleOrder = parseRuleOrder(template); - return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder); + return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder, dropImports, dropQueries); } } @@ -72,23 +72,35 @@ public class RuleFileFactory { } - private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder) { + private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder, boolean dropImports, boolean dropQueries) { - Set additionalRuleTypes = bluePrint.ruleClasses() + Set additionalRuleTypes = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleType) .filter(ruleType -> !ruleOrder.contains(ruleType)) .collect(Collectors.toSet()); StringBuilder sb = new StringBuilder(); - sb.append(bluePrint.imports()); + if (!dropImports) { + sb.append(bluePrint.getImports()); + sb.append("\n\n"); + } else { + //todo: this is hacked to enable compiling the rules without imports + sb.append("package drools"); + sb.append("\n\n"); + } + sb.append(bluePrint.getGlobals()); sb.append("\n\n"); - sb.append(bluePrint.globals()); + sb.append("//------------------------------------ declarations ------------------------------------"); sb.append("\n\n"); - sb.append("//------------------------------------ queries ------------------------------------"); - sb.append("\n\n"); - sb.append(bluePrint.queries()); + sb.append(bluePrint.getDeclarations().isEmpty() ? "" : bluePrint.getDeclarations()); sb.append("\n\n"); + if (!dropQueries) { + sb.append("//------------------------------------ queries ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getQueries()); + sb.append("\n\n"); + } for (RuleType ruleBlockType : ruleOrder) { if (ruleBlockType.isWildCard()) { additionalRuleTypes.stream() @@ -101,6 +113,9 @@ public class RuleFileFactory { } writeRuleClass(bluePrint, ruleBlockType, sb); } + sb.append("//------------------------------------ functions ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getFunctions().isEmpty() ? "" : bluePrint.getFunctions()); return sb.toString().trim() + "\n"; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java similarity index 50% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index 3e17b0db..db0ece77 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -1,30 +1,35 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; +import org.drools.drl.ast.descr.FunctionDescr; import org.drools.drl.ast.descr.ImportDescr; import org.drools.drl.ast.descr.PackageDescr; import org.drools.drl.ast.descr.RuleDescr; import org.drools.drl.parser.DrlParser; import org.kie.internal.builder.conf.LanguageLevelOption; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicDeclaration; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicFunction; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -45,10 +50,19 @@ public class RuleFileParser { @SneakyThrows public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString) { + return buildBluePrintFromRulesString(rulesString, false); + } + + + @SneakyThrows + public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString, boolean removeImports) { + DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); PackageDescr packageDescr = parser.parse(false, rulesString); StringBuilder queryBuilder = new StringBuilder(); List allRules = new LinkedList<>(); + List functions = new LinkedList<>(); + List declarations = new LinkedList<>(); for (RuleDescr rule : packageDescr.getRules()) { if (rule.isQuery()) { queryBuilder.append(rulesString, rule.getStartCharacter(), rule.getEndCharacter()); @@ -57,12 +71,30 @@ public class RuleFileParser { } allRules.add(BasicRule.fromRuleDescr(rule, rulesString)); } - String imports = rulesString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + for (FunctionDescr function : packageDescr.getFunctions()) { + functions.add(BasicFunction.fromFunctionDescr(function, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getEnumDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getClassAndEnumDeclarationDescrs()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + String imports = ""; + if (!removeImports && !packageDescr.getImports().isEmpty()) { + imports = rulesString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -70,7 +102,7 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses); + return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses, declarations, functions); } @@ -89,11 +121,17 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); + if (rulesPerUnit.keySet() != null) { + + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); + } else { + return new ArrayList<>(); + } + } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java similarity index 77% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java index c002fed3..afbdff26 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java @@ -1,15 +1,16 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.File; import java.io.FileOutputStream; import java.nio.charset.StandardCharsets; import java.util.List; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -30,7 +31,7 @@ public class RuleFileMigrator { //replaceRules(ruleFileBluePrint, combinedBluePrint); replaceRuleIdentifiers(combinedBluePrint, ruleFileBluePrint); - String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); + String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java similarity index 76% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java index fc8e9026..3889fad1 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.FileOutputStream; import java.io.IOException; @@ -9,16 +9,15 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -35,7 +34,6 @@ public class RuleIdentifierMigrator { @SneakyThrows public void migrateAllRuleIdentifiers() { - List parsedRecords = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); RuleFileBluePrint bluePrint = getBluePrint(ApplicationType.RM); // migrateIdentifier(RuleIdentifier.fromString("PII.10.0"), RuleIdentifier.fromString("CBI.20.0"), bluePrint, parsedRecords); @@ -46,20 +44,17 @@ public class RuleIdentifierMigrator { bluePrint = migrateMatchedRuleForAllRules(bluePrint); - String ruleString = RuleFileFactory.buildRuleString(bluePrint); + String ruleString = RuleFileFactory.buildRuleString(bluePrint, false, false); try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) { out.write(ruleString.getBytes(StandardCharsets.UTF_8)); } - String csvString = OldRulesParser.formatAsCsv(parsedRecords); - try (var out = new FileOutputStream("/tmp/old_rules_with_translations.csv")) { - out.write(csvString.getBytes(StandardCharsets.UTF_8)); - } } + //todo: introduce functions and declarations private static RuleFileBluePrint migrateMatchedRuleForAllRules(RuleFileBluePrint bluePrint) { - List migratedRules = bluePrint.ruleClasses() + List migratedRules = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) @@ -67,7 +62,12 @@ public class RuleIdentifierMigrator { .flatMap(Collection::stream) .map(RuleIdentifierMigrator::migrateMatchedRule) .toList(); - RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.imports(), bluePrint.globals(), bluePrint.queries(), new LinkedList<>()); + RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.getImports(), + bluePrint.getGlobals(), + bluePrint.getQueries(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>()); migratedRules.forEach(migratedBluePrint::addRule); return migratedBluePrint; } @@ -118,7 +118,7 @@ public class RuleIdentifierMigrator { } - public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List records) { + public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint) { BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()) .orElseThrow().findRuleUnitByInteger(oldIdentifier.unit()) @@ -132,9 +132,6 @@ public class RuleIdentifierMigrator { bluePrint.removeRule(oldIdentifier); bluePrint.addRule(newRule); - records.stream() - .filter(record -> record.translatesTo().contains(oldIdentifier)) - .forEach(record -> replaceOldIdentifier(oldIdentifier, newIdentifier, record.translatesTo())); } @@ -152,7 +149,7 @@ public class RuleIdentifierMigrator { } - public record Context(RuleFileBluePrint bluePrint, List records) { + public record Context(RuleFileBluePrint bluePrint) { } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java similarity index 73% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java index 8fdac0d7..53d1122c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import lombok.Getter; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java new file mode 100644 index 00000000..4e3acb5c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicDeclaration { + + @EqualsAndHashCode.Include + String declarationFullTypeName; + String body; + int line; + + + public static BasicDeclaration fromDeclarationDescription(AbstractClassTypeDeclarationDescr declaration, String rulesString) { + + String declarationFullTypeName = declaration.getFullTypeName(); + String body = rulesString.substring(declaration.getStartCharacter(), declaration.getEndCharacter()); + int line = declaration.getLine(); + + return new BasicDeclaration(declarationFullTypeName, body, line); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java new file mode 100644 index 00000000..1793d589 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java @@ -0,0 +1,36 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.FunctionDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicFunction { + + @EqualsAndHashCode.Include + String functionName; + String returnType; + String body; + int line; + + + public static BasicFunction fromFunctionDescr(FunctionDescr function, String rulesString) { + + String functionName = function.getName(); + String returnType = function.getReturnType(); + String body = rulesString.substring(function.getStartCharacter(), function.getEndCharacter()); + int line = function.getLine(); + + return new BasicFunction(functionName, returnType, body, line); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java similarity index 80% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java index 9d77bbb6..cc5c0fe6 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java @@ -1,8 +1,8 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; public record BasicRule(RuleIdentifier identifier, String name, String code) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java index a6cdb0ab..bb0e2d79 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java index 40468cc0..e70acc0f 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.LinkedList; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java similarity index 85% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 2b2ae91f..28b2663c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Collection; import java.util.Collections; @@ -10,7 +10,29 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -public record RuleFileBluePrint(String imports, String globals, String queries, List ruleClasses) { +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RuleFileBluePrint { + + String imports; + String globals; + String queries; + List ruleClasses; + List declarations; + List functions; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -24,7 +46,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, ruleClass.ruleUnits().remove(ruleUnit); } if (ruleClass.ruleUnits().isEmpty()) { - ruleClasses().remove(ruleClass); + this.ruleClasses.remove(ruleClass); } })); return wasRemoved.get(); @@ -41,7 +63,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRules() { - return ruleClasses().stream() + return this.ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -101,7 +123,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRuleIdentifiers() { - return ruleClasses().stream() + return ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -122,10 +144,16 @@ public record RuleFileBluePrint(String imports, String globals, String queries, } + public void setImports(String newImports) { + + this.imports = newImports; + } + + public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>()); - ruleClasses().stream() + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) .flatMap(ruleUnit -> ruleUnit.rules() diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java similarity index 97% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java index 569412e4..3fce64a7 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Arrays; import java.util.Objects; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java similarity index 95% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java index ffddc766..fcacf44d 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Map; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java similarity index 54% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java index ec23ad27..a2b84ba3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java similarity index 96% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java index 3d3ccac6..8efecd93 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.utils; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.utils; import java.io.File; import java.io.FileInputStream; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java new file mode 100644 index 00000000..32aaf4fc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -0,0 +1,169 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class RuleBuilderService { + + //todo: make this configurable + private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), + new RuleType("MAN"), + new RuleType("X"), + new RuleType("DICT"), + new RuleType("FA"), + new RuleType("LDS"))); + + + public RuleBuilderModel getRuleBuilderModel() { + + RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + + ruleBuilderModel.setWhenClauses(Collections.emptyList()); + ruleBuilderModel.setThenConditions(Collections.emptyList()); + + return ruleBuilderModel; + } + + + public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + log.info("Starting to remove system rules from ruleFile"); + removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); + log.info("Finished removing system rules for ruleFile"); + + return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); + } + + + public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { + + log.info("starting to merge user rules update with system rules"); + RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + mergedRuleFileBlueprint.getRuleClasses() + .forEach(ruleClass -> { + if (systemRules.stream() + .map(RuleType::name) + .toList().contains(ruleClass.ruleType().name())) { + throw new RuntimeException("No system rule updates allowed in user rule update."); + } + }); + removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getRuleClasses() + .stream() + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .flatMap(ruleUnit -> ruleUnit.rules() + .stream())) + .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); + mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() + .stream(), + mergedRuleFileBlueprint.getFunctions() + .stream()) + .toList()); + mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() + .stream(), + mergedRuleFileBlueprint.getDeclarations() + .stream()) + .toList()); + log.info("finished merging user rules update with system rules"); + RuleMergingResult mergingResult = RuleMergingResult.builder() + .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) + .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) + .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .build(); + + return mergingResult; + } + + + private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { + + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + removeAllRulesExceptSystemRules(ruleFileBluePrint); + ruleFileBluePrint.getRuleClasses() + .forEach(ruleClass -> { + if (!systemRuleNames.contains(ruleClass.ruleType().name())) { + throw new RuntimeException("there was an error removing all rules except system rules"); + } + }); + } + + + private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + + List rules = new ArrayList(); + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + + ruleFileBluePrint.getRuleClasses() + .stream() + .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) + .forEach(rule -> rules.add(rule)); + rules.forEach(ruleIdentifier -> { + ruleFileBluePrint.removeRule(ruleIdentifier); + }); + } + + + private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { + + removeSystemRules(ruleFileBluePrint); + for (RuleType systemRule : systemRules) { + List remainingSystemRules = new ArrayList(); + ruleFileBluePrint.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(remainingSystemRules::add)); + if (!remainingSystemRules.isEmpty()) { + throw new RuntimeException("There was an error removing the system rules from the file"); + } + } + } + + + private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { + + for (RuleType systemRule : systemRules) { + List rules = new ArrayList(); + ruleFileBluePrintExisting.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(rules::add)); + rules.forEach(ruleUnit -> { + ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); + }); + ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index 9b983752..e27361be 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -69,51 +69,64 @@ public class DroolsValidationService { private DroolsValidation buildCustomDroolsValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation(); + DroolsValidation customValidation = ruleCompilationResult.getDroolsValidation(); - addSyntaxDeprecatedWarnings(ruleFileBluePrint, customValidation); + addSyntaxDeprecatedWarnings(ruleCompilationResult, customValidation); - addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxErrorMessages(ruleFileType, ruleCompilationResult, customValidation); if (redactionServiceSettings.isRuleExecutionSecured()) { - addBlacklistErrorMessages(ruleFileBluePrint, customValidation); + addBlacklistErrorMessages(ruleCompilationResult, customValidation); } return customValidation; } - private void addSyntaxDeprecatedWarnings(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxDeprecatedWarnings(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { // find deprecated elements in the ruleFileBluePrint - DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); + DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleCompilationResult); if (warningMessageForImports != null) { customValidation.getDeprecatedWarnings().add(warningMessageForImports); } + customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleCompilationResult)); + } + + + private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleCompilationResult ruleCompilationResult) { + + return DroolsSyntaxDeprecatedWarnings.builder() + .message("global ComponentMappingService " + + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL + + "\n is missing from the rules, consider adding it, as it will be required in future versions!") + .line(ruleCompilationResult.getGlobalsLine()) + .column(0) + .build(); customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } - private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { + private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleCompilationResult ruleCompilationResult) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { - String imports = ruleFileBluePrint.getImports(); + String imports = ruleCompilationResult.getImports(); SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation(); List matches = classesSearchImplementation.getMatches(imports); if (!matches.isEmpty()) { String sb = "Following imports are deprecated: \n" + matches.stream() .map(m -> imports.substring(m.startIndex(), m.endIndex())) .collect(Collectors.joining("\n")); - return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build(); + return DroolsSyntaxDeprecatedWarnings.builder().line(ruleCompilationResult.getImportLine()).column(0).message(sb).build(); } } return null; } - private List getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { + private List getWarningsForDeprecatedRules(RuleCompilationResult ruleCompilationResult) { List warningMessages = new ArrayList<>(); @@ -122,7 +135,7 @@ public class DroolsValidationService { SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation(); Map deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap(); - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { List matches = methodsSearchImplementation.getMatches(basicRule.getCode()); @@ -146,32 +159,32 @@ public class DroolsValidationService { } - private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { - RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { - case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); - case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + RuleCompilationResult baseRuleCompilationResult = switch (ruleFileType) { + case ENTITY -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); + case COMPONENT -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseComponentRuleFileString()); }; - if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) { + if (!importsAreValid(baseRuleCompilationResult, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getImportLine()) + .line(ruleCompilationResult.getImportLine()) .column(0) - .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports())) + .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleCompilationResult.getImports())) .build()); } - if (!ruleFileBluePrint.getGlobals().contains(baseRuleFileBluePrint.getGlobals())) { + if (!ruleCompilationResult.getGlobals().contains(baseRuleCompilationResult.getGlobals())) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) - .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals())) + .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleCompilationResult.getGlobals())) .build()); } - baseRuleFileBluePrint.getQueries() + baseRuleCompilationResult.getQueries() .forEach(basicQuery -> { - if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { + if (!validateQueryIsPresent(basicQuery, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(basicQuery.getLine()) @@ -182,7 +195,7 @@ public class DroolsValidationService { }); if (ruleFileType.equals(RuleFileType.ENTITY)) { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; - if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { + if (!validateAgendaGroupIsPresent(ruleCompilationResult, requiredAgendaGroup)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(0) @@ -194,18 +207,18 @@ public class DroolsValidationService { } - private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) { + private boolean validateAgendaGroupIsPresent(RuleCompilationResult ruleCompilationResult, String agendaGroupName) { - return ruleFileBluePrint.streamAllRules() + return ruleCompilationResult.streamAllRules() .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName)); } - private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) { + private boolean importsAreValid(RuleCompilationResult baseRuleCompilationResult, RuleCompilationResult ruleCompilationResult) { // imports may shrink, but not add anything new! - Set baseImports = baseRuleFileBluePrint.getImportSplitByKeyword(); - Set imports = ruleFileBluePrint.getImportSplitByKeyword(); + Set baseImports = baseRuleCompilationResult.getImportSplitByKeyword(); + Set imports = ruleCompilationResult.getImportSplitByKeyword(); Set additionalImports = Sets.difference(imports, baseImports); @@ -214,15 +227,15 @@ public class DroolsValidationService { } - private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) { + private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleCompilationResult ruleCompilationResult) { - return ruleFileBluePrint.getQueries() + return ruleCompilationResult.getQueries() .stream() .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode())); } - private void addBlacklistErrorMessages(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addBlacklistErrorMessages(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { List blacklistErrorMessages = new ArrayList<>(); @@ -234,14 +247,14 @@ public class DroolsValidationService { // check also the imports DroolsBlacklistErrorMessage blacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, - ruleFileBluePrint.getImports(), - ruleFileBluePrint.getImportLine()); + ruleCompilationResult.getImports(), + ruleCompilationResult.getImportLine()); if (blacklistErrorMessage != null) { blacklistErrorMessages.add(blacklistErrorMessage); } // check the rules - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { DroolsBlacklistErrorMessage ruleBlacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java similarity index 53% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index cd6a953e..616cabde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -22,21 +22,48 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.RuleType; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @UtilityClass -public class RuleFileParser { +public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { + + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); + List allRules = new LinkedList<>(); + List allQueries = new LinkedList<>(); + PackageDescr packageDescr = new PackageDescr(); + String imports = ""; + String globals = ""; + List ruleClasses = buildRuleClasses(allRules); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + @SneakyThrows - public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); @@ -52,12 +79,15 @@ public class RuleFileParser { } } - String imports = ruleString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + String imports = ""; + if (!removedImports) { + imports = ruleString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -65,21 +95,81 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + + @SneakyThrows + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { + + return buildRuleCompilationResultFromRuleString(ruleString,false); + } + + + @SneakyThrows + public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { + + StringBuilder ruleStringBuilder = new StringBuilder(); + + // Append imports + ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); + + // Append globals + if (!bluePrint.getGlobals().isEmpty()) { + ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); + } + + // Append queries + for (BasicQuery query : bluePrint.getQueries()) { + ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); + } + + // Append rules + for (RuleClass ruleClass : bluePrint.getRuleClasses()) { + ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); + } + + // Return the final rule string + return ruleStringBuilder.toString().trim(); + } + + + private String buildQueryString(BasicQuery query) { + + return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; + } + + + private String buildRuleString(RuleClass ruleClass) { + + StringBuilder ruleBuilder = new StringBuilder(); + + // Use RuleType to distinguish between different rule types, if needed + ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); + + for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { + for (BasicRule rule : ruleUnit.rules()) { + // Assuming BasicRule has a method to retrieve the condition as a string + ruleBuilder.append(rule.getCode()).append("\n"); + } + } + ruleBuilder.append("end"); + + return ruleBuilder.toString(); } @@ -126,6 +216,7 @@ public class RuleFileParser { private List buildRuleClasses(List allRules) { + //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv diff --git a/redaction-service-v1/rules-management/src/main/resources/order_template.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/order_template.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java new file mode 100644 index 00000000..1996af83 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -0,0 +1,149 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RuleBuilderTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); + private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); + private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + + @Autowired + RuleBuilderService ruleBuilderService; + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import(LayoutParsingServiceProcessorConfiguration.class) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RuleBuilderTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + public void removeSystemRulesTest() { + + String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); + String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + + Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + + Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + } + + + @Test + public void mergeUserRulesUpdateTest() { + + RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); + String mergedRules = mergingResult.getMergedRules(); + RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); + ruleFileBluePrintUserRulesUpdate.getRuleClasses() + .forEach(ruleClass -> { + Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); + }); + + } + + + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getRuleClasses() + .stream() + .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) + .collect(Collectors.toList()).size() == 0; + } + + + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getImports().isEmpty(); + } + + + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getQueries().isEmpty(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java index cac2af47..2d95017d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java @@ -11,10 +11,10 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.core.io.ClassPathResource; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index 4db674a4..0bd9ecec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -22,10 +22,10 @@ import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; import lombok.SneakyThrows; @@ -232,11 +232,11 @@ class DroolsValidationServiceTest { if (droolsValidation.isCompiled()) { continue; } - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(rulesString); + RuleCompilationResult baseRuleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); - rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); - rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + rulesString = rulesString.replace(ruleCompilationResult.getImports(), baseRuleCompilationResult.getImports()); + rulesString = rulesString.replace(ruleCompilationResult.getGlobals(), baseRuleCompilationResult.getGlobals()); try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); @@ -457,9 +457,9 @@ class DroolsValidationServiceTest { end """; - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled()); + assertFalse(ruleCompilationResult.getDroolsValidation().isCompiled()); } } \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java similarity index 91% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java index f8d25b6a..fa77a92b 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,10 +27,10 @@ public class DroolsCompilationTest { @Test public void testValidateRuleSyntax() throws IOException { - URL rmURL = Resources.getResource("all_redact_manager_rules.drl"); + URL rmURL = Resources.getResource("rulesmanagement/all_redact_manager_rules.drl"); String rmRule = Resources.toString(rmURL, StandardCharsets.UTF_8); - URL dmURL = Resources.getResource("all_rules_documine.drl"); + URL dmURL = Resources.getResource("rulesmanagement/all_rules_documine.drl"); String dmRule = Resources.toString(dmURL, StandardCharsets.UTF_8); assertTrue(validateRuleSyntax(rmRule).isValid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java new file mode 100644 index 00000000..577f996a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; + +public class RuleFileBluePrintMergingTest { + + @Test + public void testBothRuleFilesCanBeMerged() { + + RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); + assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); + } + +} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java similarity index 93% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java index c3629d9c..be926def 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import java.io.File; import java.nio.file.Files; @@ -8,7 +8,7 @@ import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.migration.RuleFileMigrator; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java similarity index 94% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java index b6b85cef..ce84ebbf 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -19,10 +19,10 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl new file mode 100644 index 00000000..16ea73d2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl @@ -0,0 +1,50 @@ +package drools +//------------------------------------ Table extraction rules ------------------------------------ + +// Rule unit: TAB.0 +rule "TAB.0.0: Changed Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Changed Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +// Rule unit: TAB.6 +rule "TAB.6.1: Changed Targeted cell extraction (Experimental Stop date)" + when + $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2")) + $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList() + TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList() + $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList() + then + entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date")); + end + diff --git a/redaction-service-v1/rules-management/build.gradle.kts b/redaction-service-v1/rules-management/build.gradle.kts index 2aaea40c..b005ed7e 100644 --- a/redaction-service-v1/rules-management/build.gradle.kts +++ b/redaction-service-v1/rules-management/build.gradle.kts @@ -27,10 +27,10 @@ sourceSets { } dependencies { - implementation(project(":redaction-service-server-v1")) + implementation(project(mapOf("path" to ":redaction-service-server-v1"))) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") - + implementation(project(":redaction-service-server-v1")) implementation("com.github.javaparser:javaparser-core:3.25.3") implementation("org.drools:drools-drl-parser:8.41.0.Final") implementation("org.apache.commons:commons-csv:1.10.0") diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java index 7267bd20..6ab58cf3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java @@ -17,12 +17,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; @@ -72,7 +71,6 @@ public class Main { Set identifiers = switch (type) { case "list", "l" -> RuleIdentifier.fromListOfIdentifiersString(input); case "file", "f" -> RuleFileParser.parseRuleIdentifiersFromFile(input); - case "old", "o" -> OldRulesParser.translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(input), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid", cmd.getOptionValue("t"))); }; ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType); @@ -102,7 +100,6 @@ public class Main { String type = cmd.hasOption("t") ? cmd.getOptionValue("t") : "f"; Map> identifiersPerFile = switch (type) { case "file", "f" -> parseIdentifiersFromFiles(Path.of(cmd.getOptionValue("input"))); - case "old", "o" -> translateIdentifiersFromOldFiles(Path.of(cmd.getOptionValue("input")), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid in combination with the \"recursive\" flag", cmd.getOptionValue("t"))); }; identifiersPerFile.keySet() @@ -137,13 +134,6 @@ public class Main { } - @SneakyThrows - private static Map> translateIdentifiersFromOldFiles(Path inputDirectory, ApplicationType applicationType) { - - return RuleFileIO.streamAllRuleFilesInDirectory(inputDirectory) - .collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, applicationType))); - } - @SneakyThrows private static Map> parseIdentifiersFromFiles(Path inputDirectory) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java deleted file mode 100644 index af844a6d..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -public class RuleManagementResources { - - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_redact_manager_rules.drl"); - } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl"); - } - - - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers.txt"); - } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers_dm.txt"); - } - } - - - public static InputStream getTemplateInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("order_template.txt"); - } - - - public static InputStream getOldRulesCsvInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("old_rules_with_translations.csv"); - } - - - @SneakyThrows - public static String createTempOldRulesCsv(String formattedAsCsv) { - - File csvFile = File.createTempFile("old_rules_with_translations-", ".csv"); - try (var out = new FileOutputStream(csvFile)) { - out.write(formattedAsCsv.getBytes(StandardCharsets.UTF_8)); - } - return csvFile.toString(); - } - -} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java deleted file mode 100644 index ae0f3134..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java +++ /dev/null @@ -1,203 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; -import org.drools.drl.ast.descr.PackageDescr; -import org.drools.drl.ast.descr.RuleDescr; -import org.drools.drl.parser.DrlParser; -import org.kie.internal.builder.conf.LanguageLevelOption; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.OldRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; - -@UtilityClass -public class OldRulesParser { - - static List HEADERS = List.of("id", "old rule names", "old rule code", "translates to"); - - - @SneakyThrows - public void createSetOfTranslatedRuleIdentifiersForEachFileInTheCsv() { - - List allHeaders = List.of(); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - - } - - - public Set translateOldRulesFileToNewIdentifiers(File oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesFileToNewIdentifiers(oldRulesFile.toString(), applicationType); - } - - - public Set translateOldRulesFileToNewIdentifiers(String oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(oldRulesFile), applicationType); - } - - - public Set translateEscapedOldRulesStringToNewIdentifiers(String escapedOldRulesString, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.unescapeAndUnWrap(escapedOldRulesString), applicationType); - } - - - public Set translateOldRulesStringToNewIdentifiers(String oldRulesString, ApplicationType applicationType) { - - List oldRules = parseOldRules(oldRulesString); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - Map> translationPairs = new HashMap<>(); - for (OldRule oldRule : oldRules) { - List translatedIdentifiers = records.stream() - .filter(oldRulesCsvRecord -> oldRulesCsvRecord.code().equals(oldRule.code())) - .map(OldRulesCsvRecord::translatesTo) - .findAny() - .orElse(Collections.emptyList()); - translationPairs.put(oldRule, translatedIdentifiers); - } - boolean allTranslated = true; - for (OldRule oldRule : translationPairs.keySet()) { - if (translationPairs.get(oldRule).isEmpty()) { - allTranslated = false; - records.add(new OldRulesCsvRecord(records.size(), oldRule.code(), oldRule.name(), Collections.emptyList())); - System.out.printf( - "Rule %s has not been translated yet! %nIt has been added to the bottom of the old_rules_with_translations.csv file. To continue, translate the rule and enter its identifier in the csv, then try again!%n", - oldRule.name()); - } - } - if (!allTranslated) { - String formattedAsCsv = formatAsCsv(records); - String fileLocation = RuleManagementResources.createTempOldRulesCsv(formattedAsCsv); - System.out.printf("CSV File with updated values is located at %s%n", fileLocation); - throw new IllegalArgumentException("Non translated Rule found!"); - } - RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); - - translationPairs.forEach((key, value) -> System.out.printf("Rule %s will be translated to %s \n", - key.name(), - String.format("%s: %s", - value, - value.stream() - .map(bluePrint::findRuleByIdentifier) - .flatMap(Collection::stream) - .map(BasicRule::name) - .toList()))); - - return Stream.concat(// - Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),// - translationPairs.values() - .stream() - .flatMap(Collection::stream)) - .map(ruleIdentifier -> new RuleIdentifier(ruleIdentifier.type(), ruleIdentifier.unit(), null)) - .collect(Collectors.toSet()); - } - - - @SneakyThrows - public String formatAsCsv(List records) { - - StringWriter sw = new StringWriter(); - CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS.toArray(String[]::new)).build(); - try (CSVPrinter printer = new CSVPrinter(sw, csvFormat)) { - for (OldRulesCsvRecord record : records) { - printer.printRecord(Stream.of(record.id, RuleFileIO.escapeAndWrap(record.names), RuleFileIO.escapeAndWrap(record.code), record.translatesTo)); - } - } - return sw.toString(); - } - - - @SneakyThrows - private List parseOldRules(String oldRuleString) { - - DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); - PackageDescr packageDescr = parser.parse(false, oldRuleString); - List oldRules = new LinkedList<>(); - for (RuleDescr rule : packageDescr.getRules()) { - if (!rule.isRule()) { - continue; - } - oldRules.add(OldRule.fromRuleDescr(rule, oldRuleString)); - } - return oldRules; - } - - - @SneakyThrows - public List getOldRulesCsvRecords(InputStream allRulesCsvInputStream) { - - List parsedRecords = new LinkedList<>(); - try (Reader reader = new InputStreamReader(allRulesCsvInputStream)) { - Iterable records = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord().parse(reader); - for (CSVRecord record : records) { - String[] values = record.values(); - parsedRecords.add(new OldRulesCsvRecord(Long.parseLong(values[0]), - RuleFileIO.unescapeAndUnWrap(values[2]), - RuleFileIO.unescapeAndUnWrap(values[1]), - parseRuleIdentifiers(values[3]))); - } - } - return parsedRecords; - } - - - private List parseRuleIdentifiers(String value) { - - String cleanedValue = value; - if (cleanedValue.startsWith("[")) { - cleanedValue = cleanedValue.substring(1); - } - if (cleanedValue.endsWith("]")) { - cleanedValue = cleanedValue.substring(0, cleanedValue.length() - 1); - } - if (cleanedValue.isEmpty() || cleanedValue.isBlank()) { - return Collections.emptyList(); - } - List ruleIdentifiers = new LinkedList<>(); - for (String identifier : cleanedValue.split(", ")) { - ruleIdentifiers.add(RuleIdentifier.fromString(identifier)); - } - return ruleIdentifiers; - } - - - private List parseBooleanList(String[] values) { - - return Arrays.stream(values) - .map(Boolean::parseBoolean) - .toList(); - } - - - public record OldRulesCsvRecord(long id, String code, String names, List translatesTo) { - - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java deleted file mode 100644 index 559101ac..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; - -public class RuleFileBluePrintMergingTest { - - @Test - public void testBothRuleFilesCanBeMerged() { - - RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); - assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java deleted file mode 100644 index b78ab534..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -class OldRulesParserTest { - - @Test - @SneakyThrows - @Disabled - public void translateOldRulesToNewIdentifiersTest() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("efsa_regulation_rules.txt")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateEscapedOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @SneakyThrows - @Disabled - public void translateMoreOldRules() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("basf/demo/table_demo.drl")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @Disabled - public void printTranslationsTest() { - - List records = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")) - .forEach(type -> { - List rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type) - .orElseThrow().ruleUnits(); - rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records)); - }); - } - - - private void printOldRulesThatTranslatesToNewRule(RuleUnit ruleUnit, List records) { - - if (ruleUnit.rules().isEmpty()) { - System.out.println("Rule unit empty, skipping!\n"); - return; - } - RuleIdentifier identifier = ruleUnit.rules() - .get(0).identifier(); - RuleIdentifier unitIdentifier = new RuleIdentifier(identifier.type(), identifier.unit(), null); - String oldNames = records.stream() - .filter(r -> r.translatesTo() - .stream() - .anyMatch(i -> i.matches(unitIdentifier))) - .map(OldRulesParser.OldRulesCsvRecord::names) - .map(OldRulesParserTest::removeIdFromName) - .distinct() - .collect(Collectors.joining(", ")); - System.out.println(unitIdentifier.toRuleUnitString() + " " + ruleUnit.rules() - .stream() - .map(BasicRule::name) - .collect(Collectors.joining(", "))); - System.out.println("translate from"); - System.out.println(oldNames); - System.out.println(); - System.out.println(); - } - - - private static String removeIdFromName(String name) { - - String[] values = name.split(":"); - return String.join(":", Arrays.copyOfRange(values, 1, values.length)); - } - - - @Test - @Disabled - @SneakyThrows - public void findAllCustomerRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("pilot/EFSA_sanitisation_GFL_v1").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.RM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.RM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void findAllRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("business-logic/Syngenta_RSS").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.DM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.DM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void translateDossierTemplatesV2() { - -// String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/RED/dossier-templates-v2/"; -// Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), Files.walk(Path.of(dossierTemplatesRepo + "docu")), Files.walk(Path.of(dossierTemplatesRepo + "qa"))) - String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/"; - Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), - Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) - // - ) - .flatMap(Function.identity())// - .filter(path -> path.getFileName().toString().equals("rules.drl"))// - .map(Path::toFile)// - .forEach(this::translateOldRuleFile); - - } - - - @SneakyThrows - private void translateOldRuleFile(File oldRulesFile) { - - Set identifiers = OldRulesParser.translateOldRulesFileToNewIdentifiers(oldRulesFile, ApplicationType.RM); - String newRulesString = RuleFileFactory.createFileFromIdentifiers(identifiers, ApplicationType.RM); - String result = RuleFileIO.escapeAndWrap(newRulesString); - - try (var out = new FileOutputStream(oldRulesFile)) { - out.write(result.getBytes(StandardCharsets.UTF_8)); - } - } - -} \ No newline at end of file