From de7ea74ccf11e5dc4a5cfbfb1bb3f78bc6102197 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Sep 2024 10:24:59 +0200 Subject: [PATCH 001/108] RED-9472: seperation of system rules added endpoints for seperating and readding system rules working on service RED-9472: seperation of system rules refactored endpoints. now removing system rules on download merging user rules with systemrules on upload RED-9472: seperation of system rules RED-9472: seperation of system rules RED-9472: seperation of system rules refactored two have a rulefileblueprint and a rulefileblueprint for validationresult changed the flow of merging and seperating system and user rules added functions and declarations to rule file parsing RED-9472: seperation of system rules RED-9472: seperation of system rules introduced tests working on fixing bug with merging rules RED-9472: seperation of system rules introduced tests RED-9472: seperation of system rules fixed circular depdency RED-9472: seperation of system rules updated persistence service version RED-9472: seperation of system rules refactored rules refactored pcackages RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed some sonarqube errors RED-9472: seperation of system rules fixed pmd error RED-9472: seperation of system rules added deploy stage to gitlab pipeline RED-9472: seperation of system rules RED-9472: seperation of system rules fixed a mistake RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse --- .gitlab-ci.yml | 18 ++ .../redaction-service-api-v1/build.gradle.kts | 3 +- .../v1/resources/RuleBuilderResource.java | 17 +- .../build.gradle.kts | 3 +- .../controller/RuleBuilderController.java | 96 ++++++++- .../v1/server/model/RuleMergingResult.java | 24 +++ .../model/document/nodes/SuperSection.java | 3 - ...ePrint.java => RuleCompilationResult.java} | 67 +++++- .../RuleManagementResources.java | 34 +++ .../DefaultRuleIdentifiersFactory.java | 8 +- .../factory/RuleFileFactory.java | 51 +++-- .../factory/RuleFileParser.java | 82 +++++-- .../migration/RuleFileMigrator.java | 16 +- .../migration/RuleIdentifierMigrator.java | 45 ++-- .../models/ApplicationType.java | 2 +- .../models/BasicDeclaration.java | 34 +++ .../rulesmanagement/models/BasicFunction.java | 36 ++++ .../rulesmanagement}/models/BasicRule.java | 4 +- .../rulesmanagement}/models/OldRule.java | 2 +- .../rulesmanagement}/models/RuleClass.java | 2 +- .../models/RuleFileBluePrint.java | 42 +++- .../models/RuleIdentifier.java | 2 +- .../rulesmanagement}/models/RuleType.java | 2 +- .../rulesmanagement}/models/RuleUnit.java | 2 +- .../rulesmanagement}/utils/RuleFileIO.java | 2 +- .../v1/server/service/RuleBuilderService.java | 169 +++++++++++++++ .../drools/DroolsValidationService.java | 84 ++++---- ....java => RuleCompilationResultParser.java} | 139 +++++++++--- .../all_redact_manager_rules.drl | 0 .../rulesmanagement}/all_rules_documine.drl | 0 .../default_rule_identifiers.txt | 0 .../default_rule_identifiers_dm.txt | 0 .../old_rules_with_translations.csv | 0 .../rulesmanagement}/order_template.txt | 0 .../redaction/v1/server/RuleBuilderTest.java | 149 +++++++++++++ .../services/DroolsUpToDateTest.java | 8 +- .../services/DroolsValidationServiceTest.java | 16 +- .../DroolsCompilationTest.java | 6 +- .../RuleFileBluePrintMergingTest.java | 20 ++ .../RuleFileMigrationTest.java | 4 +- .../factory/RuleFileFactoryTest.java | 10 +- .../EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rulesmanagement}/addNewRulesHere.drl | 0 .../rulesmanagement}/demo/table_demo.drl | 0 .../rulesmanagement}/dev/Basf-Demo/rules.txt | 0 .../rulesmanagement}/dev/Flora/rules.txt | 0 .../dev/EFSA_Regulation_2021/rules.txt | 0 .../dev/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rules.txt | 0 .../EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../dev/Manual_Redaction/rules.txt | 0 .../docu/EFSA_Regulation_2021/rules.txt | 0 .../docu/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../docu/Manual_Redaction/rules.txt | 0 .../qa/EFSA_Regulation_2021/rules.txt | 0 .../qa/EFSA_Regulation_2022/rules.txt | 0 .../qa/EFSA_acceptance_test/rules.txt | 0 .../qa/EFSA_function_tests/rules.txt | 0 .../qa/EFSA_rule_test/rules.txt | 0 .../qa/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rules.txt | 0 .../qa/EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../qa/EFSA_security_test/rules.txt | 0 .../qa/EFSA_smoke_tests/rules.txt | 0 .../qa/Manual_Redaction/rules.txt | 0 .../dossier_templates_v2/qa/PII/rules.txt | 0 .../qa/case_insensitive/rules.txt | 0 .../qa/no_redaction_indicator/rules.txt | 0 .../qa/vertebrate_only/rules.txt | 0 .../efsa_regulation_rules.txt | 0 .../userrules/user_rule_update.drl | 50 +++++ .../rules-management/build.gradle.kts | 4 +- .../utility/rules/management/Main.java | 20 +- .../management/RuleManagementResources.java | 56 ----- .../translation/OldRulesParser.java | 203 ------------------ .../RuleFileBluePrintMergingTest.java | 20 -- .../translation/OldRulesParserTest.java | 176 --------------- 78 files changed, 1063 insertions(+), 668 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/{RuleFileBluePrint.java => RuleCompilationResult.java} (60%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/DefaultRuleIdentifiersFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileParser.java (50%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleFileMigrator.java (81%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleIdentifierMigrator.java (76%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/ApplicationType.java (73%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/BasicRule.java (80%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/OldRule.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleClass.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleFileBluePrint.java (85%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleIdentifier.java (97%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleType.java (95%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleUnit.java (54%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/utils/RuleFileIO.java (96%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/{RuleFileParser.java => RuleCompilationResultParser.java} (53%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_redact_manager_rules.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_rules_documine.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers_dm.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/old_rules_with_translations.csv (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/order_template.txt (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/DroolsCompilationTest.java (91%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/RuleFileMigrationTest.java (93%) rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactoryTest.java (94%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/addNewRulesHere.drl (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/demo/table_demo.drl (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dev/Basf-Demo/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dev/Flora/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_function_tests/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_rule_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_security_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/PII/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/case_insensitive/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/no_redaction_indicator/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/vertebrate_only/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/efsa_regulation_rules.txt (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62149a06..ff16ba00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,24 @@ include: ref: 'main' file: 'ci-templates/gradle_java.yml' +deploy: + stage: deploy + tags: + - dind + script: + - echo "Building with gradle version ${BUILDVERSION}" + - gradle -Pversion=${BUILDVERSION} publish + - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION} + - echo "BUILDVERSION=$BUILDVERSION" >> version.env + artifacts: + reports: + dotenv: version.env + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_COMMIT_BRANCH =~ /^release/ + - if: $CI_COMMIT_BRANCH =~ /^feature/ + - if: $CI_COMMIT_TAG + deploy JavaDoc: stage: deploy tags: diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 96bccabd..ccff795e 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,11 +4,12 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.581.0" +val persistenceServiceVersion = "2.585.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + api("com.knecon.fforesight:swagger-commons:0.7.0") } publishing { diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index 3864ba9b..4683fd5f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,13 +1,26 @@ package com.iqser.red.service.redaction.v1.resources; -import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; - import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; public interface RuleBuilderResource { @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE) RuleBuilderModel getRuleBuilderModel(); + + @PostMapping(value = "/internal-api/rules/user-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithoutSystemRules(@RequestBody SystemRulesSeperationRequest systemRulesSeperationRequest); + + + @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + ResponseEntity mergeUserUpdateRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); + } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 75fef208..277f9fde 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.581.0" +val persistenceServiceVersion = "2.585.0-RED9472.1" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" @@ -83,6 +83,7 @@ dependencies { implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") + api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 306dccbe..8f3a51dd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,27 +1,111 @@ package com.iqser.red.service.redaction.v1.server.controller; -import java.util.Collections; - +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleBlacklistErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxWarningMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @RestController +@Slf4j @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { + private final RuleBuilderService ruleBuilderService; + private final DroolsValidationService droolsValidationService; + + @Override public RuleBuilderModel getRuleBuilderModel() { - RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + return this.ruleBuilderService.getRuleBuilderModel(); + } - ruleBuilderModel.setWhenClauses(Collections.emptyList()); - ruleBuilderModel.setThenConditions(Collections.emptyList()); - return ruleBuilderModel; + @Override + public RulesResponse getRuleFileWithoutSystemRules(SystemRulesSeperationRequest systemRulesSeperationRequest) { + + RulesResponse rulesResponse = new RulesResponse(); + String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); + if (filteredRules.isEmpty()) { + throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + } + rulesResponse.setRules(filteredRules); + return rulesResponse; + } + + + @Override + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + + RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); + DroolsValidationResponse droolsValidationResponse; + RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + if (mergingResult.getMergedRules().isEmpty()) { + throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + } + try { + var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + droolsValidationResponse = DroolsValidationResponse.builder() + .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() + .stream() + .map(droolsSyntaxErrorMessage -> new RuleSyntaxErrorMessage(droolsSyntaxErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxErrorMessage.getColumn(), + droolsSyntaxErrorMessage.getMessage())) + .toList()) + .deprecatedWarnings(droolsValidation.getDeprecatedWarnings() + .stream() + .map(droolsSyntaxDeprecatedWarnings -> new RuleSyntaxWarningMessage(droolsSyntaxDeprecatedWarnings.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxDeprecatedWarnings.getColumn(), + droolsSyntaxDeprecatedWarnings.getMessage())) + .toList()) + .blacklistErrorMessages(droolsValidation.getBlacklistErrorMessages() + .stream() + .map(droolsBlacklistErrorMessage -> new RuleBlacklistErrorMessage(droolsBlacklistErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsBlacklistErrorMessage.getColumn(), + droolsBlacklistErrorMessage.getMessage())) + .toList()) + .build(); + if (!droolsValidation.isCompiled()) { + return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + } else { + // + rulesUploadResponse.setRules(mergingResult.getMergedRules()); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + } + } catch (Exception e) { + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + } + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); + } + + + private enum RuleFileType { + ENTITY, + COMPONENT } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java new file mode 100644 index 00000000..b2cb1b27 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Schema(description = "Object containing a string of Drools rules.") +public class RuleMergingResult { + + @Schema(description = "The merged rules.") + private String mergedRules; + + @Schema(description = "the length of added imports from sytemRules") + private int addedImportsOffset; + @Schema(description = "the length of added globals from sytemRules") + private int addedGlobalsOffset; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SuperSection.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SuperSection.java index 801190f1..1c0ef665 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SuperSection.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SuperSection.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes; import lombok.AccessLevel; import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.FieldDefaults; @@ -46,7 +45,6 @@ public class SuperSection extends AbstractSemanticNode { } - @Override public String toString() { @@ -86,5 +84,4 @@ public class SuperSection extends AbstractSemanticNode { return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value)); } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java similarity index 60% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index 338e58a7..c7e35460 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -18,13 +18,15 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Builder @NoArgsConstructor @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public final class RuleFileBluePrint { +public final class RuleCompilationResult { String imports; int importLine; @@ -43,9 +45,62 @@ public final class RuleFileBluePrint { } + public void dropRulesByIdentifier(RuleType ruleType) { + + log.info("removing rule with identifier {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropAllRulesExceptSystemRules(List systemRules) { + + log.info("removing all rules except the system rules"); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropImports() { + + this.imports = ""; + } + + + public void dropQueries() { + + this.queries.clear(); + } + + + public int countRuleOccurences(RuleType ruleType) { + + log.info("counting occurences of files {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + return rulesToBeRemoved.size(); + } + + + private boolean filterOutRule(RuleType ruleType, List filteredRules) { + + return !filteredRules.contains(ruleType.name()); + } + + public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replaceAll("\n", "").split("import")) + return Arrays.stream(imports.replace("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } @@ -74,7 +129,7 @@ public final class RuleFileBluePrint { public List getAllRuleIdentifiers() { return streamAllRules().map(BasicRule::getIdentifier) - .collect(Collectors.toList()); + .toList(); } @@ -94,4 +149,10 @@ public final class RuleFileBluePrint { return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']'; } + + public void addRuleClass(RuleClass ruleClass) { + + this.ruleClasses.add(ruleClass); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java new file mode 100644 index 00000000..331b1e25 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import java.io.InputStream; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; + +@SuppressWarnings("PMD") +public class RuleManagementResources { + + public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + } + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + } + + + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + } else { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + } + } + + + public static InputStream getTemplateInputStream() { + + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java index 2a8de641..c83f7f04 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java @@ -1,12 +1,12 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java index 039664bf..f6332332 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.Collections.emptySet; @@ -9,13 +9,13 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -48,17 +48,17 @@ public class RuleFileFactory { } RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers); - return buildRuleString(filteredBluePrint); + return buildRuleString(filteredBluePrint, false, false); } @SneakyThrows - public String buildRuleString(RuleFileBluePrint bluePrint) { + public String buildRuleString(RuleFileBluePrint bluePrint, boolean dropImports, boolean dropQueries) { try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) { String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8); List templateRuleOrder = parseRuleOrder(template); - return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder); + return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder, dropImports, dropQueries); } } @@ -72,23 +72,35 @@ public class RuleFileFactory { } - private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder) { + private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder, boolean dropImports, boolean dropQueries) { - Set additionalRuleTypes = bluePrint.ruleClasses() + Set additionalRuleTypes = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleType) .filter(ruleType -> !ruleOrder.contains(ruleType)) .collect(Collectors.toSet()); StringBuilder sb = new StringBuilder(); - sb.append(bluePrint.imports()); + if (!dropImports) { + sb.append(bluePrint.getImports()); + sb.append("\n\n"); + } else { + //todo: this is hacked to enable compiling the rules without imports + sb.append("package drools"); + sb.append("\n\n"); + } + sb.append(bluePrint.getGlobals()); sb.append("\n\n"); - sb.append(bluePrint.globals()); + sb.append("//------------------------------------ declarations ------------------------------------"); sb.append("\n\n"); - sb.append("//------------------------------------ queries ------------------------------------"); - sb.append("\n\n"); - sb.append(bluePrint.queries()); + sb.append(bluePrint.getDeclarations().isEmpty() ? "" : bluePrint.getDeclarations()); sb.append("\n\n"); + if (!dropQueries) { + sb.append("//------------------------------------ queries ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getQueries()); + sb.append("\n\n"); + } for (RuleType ruleBlockType : ruleOrder) { if (ruleBlockType.isWildCard()) { additionalRuleTypes.stream() @@ -101,6 +113,9 @@ public class RuleFileFactory { } writeRuleClass(bluePrint, ruleBlockType, sb); } + sb.append("//------------------------------------ functions ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getFunctions().isEmpty() ? "" : bluePrint.getFunctions()); return sb.toString().trim() + "\n"; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java similarity index 50% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index 3e17b0db..db0ece77 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -1,30 +1,35 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; +import org.drools.drl.ast.descr.FunctionDescr; import org.drools.drl.ast.descr.ImportDescr; import org.drools.drl.ast.descr.PackageDescr; import org.drools.drl.ast.descr.RuleDescr; import org.drools.drl.parser.DrlParser; import org.kie.internal.builder.conf.LanguageLevelOption; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicDeclaration; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicFunction; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -45,10 +50,19 @@ public class RuleFileParser { @SneakyThrows public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString) { + return buildBluePrintFromRulesString(rulesString, false); + } + + + @SneakyThrows + public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString, boolean removeImports) { + DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); PackageDescr packageDescr = parser.parse(false, rulesString); StringBuilder queryBuilder = new StringBuilder(); List allRules = new LinkedList<>(); + List functions = new LinkedList<>(); + List declarations = new LinkedList<>(); for (RuleDescr rule : packageDescr.getRules()) { if (rule.isQuery()) { queryBuilder.append(rulesString, rule.getStartCharacter(), rule.getEndCharacter()); @@ -57,12 +71,30 @@ public class RuleFileParser { } allRules.add(BasicRule.fromRuleDescr(rule, rulesString)); } - String imports = rulesString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + for (FunctionDescr function : packageDescr.getFunctions()) { + functions.add(BasicFunction.fromFunctionDescr(function, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getEnumDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getClassAndEnumDeclarationDescrs()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + String imports = ""; + if (!removeImports && !packageDescr.getImports().isEmpty()) { + imports = rulesString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -70,7 +102,7 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses); + return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses, declarations, functions); } @@ -89,11 +121,17 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); + if (rulesPerUnit.keySet() != null) { + + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); + } else { + return new ArrayList<>(); + } + } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java similarity index 81% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java index 94975b8e..120a2a73 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.File; import java.io.FileOutputStream; @@ -6,12 +6,12 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -33,7 +33,7 @@ public class RuleFileMigrator { //replaceRules(ruleFileBluePrint, combinedBluePrint); replaceRuleIdentifiers(combinedBluePrint, ruleFileBluePrint); - String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); + String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java similarity index 76% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java index fc8e9026..3889fad1 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.FileOutputStream; import java.io.IOException; @@ -9,16 +9,15 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -35,7 +34,6 @@ public class RuleIdentifierMigrator { @SneakyThrows public void migrateAllRuleIdentifiers() { - List parsedRecords = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); RuleFileBluePrint bluePrint = getBluePrint(ApplicationType.RM); // migrateIdentifier(RuleIdentifier.fromString("PII.10.0"), RuleIdentifier.fromString("CBI.20.0"), bluePrint, parsedRecords); @@ -46,20 +44,17 @@ public class RuleIdentifierMigrator { bluePrint = migrateMatchedRuleForAllRules(bluePrint); - String ruleString = RuleFileFactory.buildRuleString(bluePrint); + String ruleString = RuleFileFactory.buildRuleString(bluePrint, false, false); try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) { out.write(ruleString.getBytes(StandardCharsets.UTF_8)); } - String csvString = OldRulesParser.formatAsCsv(parsedRecords); - try (var out = new FileOutputStream("/tmp/old_rules_with_translations.csv")) { - out.write(csvString.getBytes(StandardCharsets.UTF_8)); - } } + //todo: introduce functions and declarations private static RuleFileBluePrint migrateMatchedRuleForAllRules(RuleFileBluePrint bluePrint) { - List migratedRules = bluePrint.ruleClasses() + List migratedRules = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) @@ -67,7 +62,12 @@ public class RuleIdentifierMigrator { .flatMap(Collection::stream) .map(RuleIdentifierMigrator::migrateMatchedRule) .toList(); - RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.imports(), bluePrint.globals(), bluePrint.queries(), new LinkedList<>()); + RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.getImports(), + bluePrint.getGlobals(), + bluePrint.getQueries(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>()); migratedRules.forEach(migratedBluePrint::addRule); return migratedBluePrint; } @@ -118,7 +118,7 @@ public class RuleIdentifierMigrator { } - public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List records) { + public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint) { BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()) .orElseThrow().findRuleUnitByInteger(oldIdentifier.unit()) @@ -132,9 +132,6 @@ public class RuleIdentifierMigrator { bluePrint.removeRule(oldIdentifier); bluePrint.addRule(newRule); - records.stream() - .filter(record -> record.translatesTo().contains(oldIdentifier)) - .forEach(record -> replaceOldIdentifier(oldIdentifier, newIdentifier, record.translatesTo())); } @@ -152,7 +149,7 @@ public class RuleIdentifierMigrator { } - public record Context(RuleFileBluePrint bluePrint, List records) { + public record Context(RuleFileBluePrint bluePrint) { } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java similarity index 73% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java index 8fdac0d7..53d1122c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import lombok.Getter; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java new file mode 100644 index 00000000..4e3acb5c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicDeclaration { + + @EqualsAndHashCode.Include + String declarationFullTypeName; + String body; + int line; + + + public static BasicDeclaration fromDeclarationDescription(AbstractClassTypeDeclarationDescr declaration, String rulesString) { + + String declarationFullTypeName = declaration.getFullTypeName(); + String body = rulesString.substring(declaration.getStartCharacter(), declaration.getEndCharacter()); + int line = declaration.getLine(); + + return new BasicDeclaration(declarationFullTypeName, body, line); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java new file mode 100644 index 00000000..1793d589 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java @@ -0,0 +1,36 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.FunctionDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicFunction { + + @EqualsAndHashCode.Include + String functionName; + String returnType; + String body; + int line; + + + public static BasicFunction fromFunctionDescr(FunctionDescr function, String rulesString) { + + String functionName = function.getName(); + String returnType = function.getReturnType(); + String body = rulesString.substring(function.getStartCharacter(), function.getEndCharacter()); + int line = function.getLine(); + + return new BasicFunction(functionName, returnType, body, line); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java similarity index 80% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java index 9d77bbb6..cc5c0fe6 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java @@ -1,8 +1,8 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; public record BasicRule(RuleIdentifier identifier, String name, String code) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java index a6cdb0ab..bb0e2d79 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java index 40468cc0..e70acc0f 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.LinkedList; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java similarity index 85% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 2b2ae91f..28b2663c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Collection; import java.util.Collections; @@ -10,7 +10,29 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -public record RuleFileBluePrint(String imports, String globals, String queries, List ruleClasses) { +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RuleFileBluePrint { + + String imports; + String globals; + String queries; + List ruleClasses; + List declarations; + List functions; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -24,7 +46,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, ruleClass.ruleUnits().remove(ruleUnit); } if (ruleClass.ruleUnits().isEmpty()) { - ruleClasses().remove(ruleClass); + this.ruleClasses.remove(ruleClass); } })); return wasRemoved.get(); @@ -41,7 +63,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRules() { - return ruleClasses().stream() + return this.ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -101,7 +123,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRuleIdentifiers() { - return ruleClasses().stream() + return ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -122,10 +144,16 @@ public record RuleFileBluePrint(String imports, String globals, String queries, } + public void setImports(String newImports) { + + this.imports = newImports; + } + + public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>()); - ruleClasses().stream() + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) .flatMap(ruleUnit -> ruleUnit.rules() diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java similarity index 97% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java index 1c863af2..904cc37c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Arrays; import java.util.Objects; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java similarity index 95% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java index ffddc766..fcacf44d 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Map; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java similarity index 54% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java index ec23ad27..a2b84ba3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java similarity index 96% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java index 3d3ccac6..8efecd93 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.utils; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.utils; import java.io.File; import java.io.FileInputStream; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java new file mode 100644 index 00000000..32aaf4fc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -0,0 +1,169 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class RuleBuilderService { + + //todo: make this configurable + private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), + new RuleType("MAN"), + new RuleType("X"), + new RuleType("DICT"), + new RuleType("FA"), + new RuleType("LDS"))); + + + public RuleBuilderModel getRuleBuilderModel() { + + RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + + ruleBuilderModel.setWhenClauses(Collections.emptyList()); + ruleBuilderModel.setThenConditions(Collections.emptyList()); + + return ruleBuilderModel; + } + + + public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + log.info("Starting to remove system rules from ruleFile"); + removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); + log.info("Finished removing system rules for ruleFile"); + + return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); + } + + + public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { + + log.info("starting to merge user rules update with system rules"); + RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + mergedRuleFileBlueprint.getRuleClasses() + .forEach(ruleClass -> { + if (systemRules.stream() + .map(RuleType::name) + .toList().contains(ruleClass.ruleType().name())) { + throw new RuntimeException("No system rule updates allowed in user rule update."); + } + }); + removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getRuleClasses() + .stream() + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .flatMap(ruleUnit -> ruleUnit.rules() + .stream())) + .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); + mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() + .stream(), + mergedRuleFileBlueprint.getFunctions() + .stream()) + .toList()); + mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() + .stream(), + mergedRuleFileBlueprint.getDeclarations() + .stream()) + .toList()); + log.info("finished merging user rules update with system rules"); + RuleMergingResult mergingResult = RuleMergingResult.builder() + .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) + .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) + .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .build(); + + return mergingResult; + } + + + private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { + + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + removeAllRulesExceptSystemRules(ruleFileBluePrint); + ruleFileBluePrint.getRuleClasses() + .forEach(ruleClass -> { + if (!systemRuleNames.contains(ruleClass.ruleType().name())) { + throw new RuntimeException("there was an error removing all rules except system rules"); + } + }); + } + + + private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + + List rules = new ArrayList(); + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + + ruleFileBluePrint.getRuleClasses() + .stream() + .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) + .forEach(rule -> rules.add(rule)); + rules.forEach(ruleIdentifier -> { + ruleFileBluePrint.removeRule(ruleIdentifier); + }); + } + + + private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { + + removeSystemRules(ruleFileBluePrint); + for (RuleType systemRule : systemRules) { + List remainingSystemRules = new ArrayList(); + ruleFileBluePrint.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(remainingSystemRules::add)); + if (!remainingSystemRules.isEmpty()) { + throw new RuntimeException("There was an error removing the system rules from the file"); + } + } + } + + + private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { + + for (RuleType systemRule : systemRules) { + List rules = new ArrayList(); + ruleFileBluePrintExisting.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(rules::add)); + rules.forEach(ruleUnit -> { + ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); + }); + ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index 6d98ae0c..df73f4b8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -28,7 +28,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplemen import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; @@ -69,70 +69,70 @@ public class DroolsValidationService { private DroolsValidation buildCustomDroolsValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation(); + DroolsValidation customValidation = ruleCompilationResult.getDroolsValidation(); - addSyntaxDeprecatedWarnings(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxDeprecatedWarnings(ruleFileType, ruleCompilationResult, customValidation); - addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxErrorMessages(ruleFileType, ruleCompilationResult, customValidation); if (redactionServiceSettings.isRuleExecutionSecured()) { - addBlacklistErrorMessages(ruleFileBluePrint, customValidation); + addBlacklistErrorMessages(ruleCompilationResult, customValidation); } return customValidation; } - private void addSyntaxDeprecatedWarnings(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxDeprecatedWarnings(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { // find deprecated elements in the ruleFileBluePrint - DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); + DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleCompilationResult); if (warningMessageForImports != null) { customValidation.getDeprecatedWarnings().add(warningMessageForImports); } - customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); + customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleCompilationResult)); if (ruleFileType.equals(RuleFileType.COMPONENT)) { - if (!ruleFileBluePrint.getGlobals().contains(ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL)) { - customValidation.getDeprecatedWarnings().add(buildComponentMappingServiceMissingMessage(ruleFileBluePrint)); + if (!ruleCompilationResult.getGlobals().contains(ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL)) { + customValidation.getDeprecatedWarnings().add(buildComponentMappingServiceMissingMessage(ruleCompilationResult)); } } } - private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleFileBluePrint ruleFileBluePrint) { + private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleCompilationResult ruleCompilationResult) { return DroolsSyntaxDeprecatedWarnings.builder() .message("global ComponentMappingService " + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL + "\n is missing from the rules, consider adding it, as it will be required in future versions!") - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) .build(); } - private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { + private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleCompilationResult ruleCompilationResult) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { - String imports = ruleFileBluePrint.getImports(); + String imports = ruleCompilationResult.getImports(); SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation(); List matches = classesSearchImplementation.getMatches(imports); if (!matches.isEmpty()) { String sb = "Following imports are deprecated: \n" + matches.stream() .map(m -> imports.substring(m.startIndex(), m.endIndex())) .collect(Collectors.joining("\n")); - return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build(); + return DroolsSyntaxDeprecatedWarnings.builder().line(ruleCompilationResult.getImportLine()).column(0).message(sb).build(); } } return null; } - private List getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { + private List getWarningsForDeprecatedRules(RuleCompilationResult ruleCompilationResult) { List warningMessages = new ArrayList<>(); @@ -141,7 +141,7 @@ public class DroolsValidationService { SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation(); Map deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap(); - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { List matches = methodsSearchImplementation.getMatches(basicRule.getCode()); @@ -165,32 +165,32 @@ public class DroolsValidationService { } - private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { - RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { - case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); - case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + RuleCompilationResult baseRuleCompilationResult = switch (ruleFileType) { + case ENTITY -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); + case COMPONENT -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseComponentRuleFileString()); }; - if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) { + if (!importsAreValid(baseRuleCompilationResult, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getImportLine()) + .line(ruleCompilationResult.getImportLine()) .column(0) - .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports())) + .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleCompilationResult.getImports())) .build()); } - if (!ruleFileBluePrint.getGlobals().contains(baseRuleFileBluePrint.getGlobals())) { + if (!ruleCompilationResult.getGlobals().contains(baseRuleCompilationResult.getGlobals())) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) - .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals())) + .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleCompilationResult.getGlobals())) .build()); } - baseRuleFileBluePrint.getQueries() + baseRuleCompilationResult.getQueries() .forEach(basicQuery -> { - if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { + if (!validateQueryIsPresent(basicQuery, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(basicQuery.getLine()) @@ -201,7 +201,7 @@ public class DroolsValidationService { }); if (ruleFileType.equals(RuleFileType.ENTITY)) { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; - if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { + if (!validateAgendaGroupIsPresent(ruleCompilationResult, requiredAgendaGroup)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(0) @@ -213,18 +213,18 @@ public class DroolsValidationService { } - private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) { + private boolean validateAgendaGroupIsPresent(RuleCompilationResult ruleCompilationResult, String agendaGroupName) { - return ruleFileBluePrint.streamAllRules() + return ruleCompilationResult.streamAllRules() .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName)); } - private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) { + private boolean importsAreValid(RuleCompilationResult baseRuleCompilationResult, RuleCompilationResult ruleCompilationResult) { // imports may shrink, but not add anything new! - Set baseImports = baseRuleFileBluePrint.getImportSplitByKeyword(); - Set imports = ruleFileBluePrint.getImportSplitByKeyword(); + Set baseImports = baseRuleCompilationResult.getImportSplitByKeyword(); + Set imports = ruleCompilationResult.getImportSplitByKeyword(); Set additionalImports = Sets.difference(imports, baseImports); @@ -233,15 +233,15 @@ public class DroolsValidationService { } - private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) { + private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleCompilationResult ruleCompilationResult) { - return ruleFileBluePrint.getQueries() + return ruleCompilationResult.getQueries() .stream() .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode())); } - private void addBlacklistErrorMessages(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addBlacklistErrorMessages(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { List blacklistErrorMessages = new ArrayList<>(); @@ -253,14 +253,14 @@ public class DroolsValidationService { // check also the imports DroolsBlacklistErrorMessage blacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, - ruleFileBluePrint.getImports(), - ruleFileBluePrint.getImportLine()); + ruleCompilationResult.getImports(), + ruleCompilationResult.getImportLine()); if (blacklistErrorMessage != null) { blacklistErrorMessages.add(blacklistErrorMessage); } // check the rules - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { DroolsBlacklistErrorMessage ruleBlacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java similarity index 53% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 375dc0f2..2a5158f3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -20,7 +20,7 @@ import com.iqser.red.service.redaction.v1.model.DroolsValidation; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; @@ -29,14 +29,41 @@ import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @UtilityClass -public class RuleFileParser { +public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { + + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); + List allRules = new LinkedList<>(); + List allQueries = new LinkedList<>(); + PackageDescr packageDescr = new PackageDescr(); + String imports = ""; + String globals = ""; + List ruleClasses = buildRuleClasses(allRules); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + @SneakyThrows - public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); @@ -52,12 +79,15 @@ public class RuleFileParser { } } - String imports = ruleString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + String imports = ""; + if (!removedImports) { + imports = ruleString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -65,21 +95,81 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + + @SneakyThrows + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { + + return buildRuleCompilationResultFromRuleString(ruleString,false); + } + + + @SneakyThrows + public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { + + StringBuilder ruleStringBuilder = new StringBuilder(); + + // Append imports + ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); + + // Append globals + if (!bluePrint.getGlobals().isEmpty()) { + ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); + } + + // Append queries + for (BasicQuery query : bluePrint.getQueries()) { + ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); + } + + // Append rules + for (RuleClass ruleClass : bluePrint.getRuleClasses()) { + ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); + } + + // Return the final rule string + return ruleStringBuilder.toString().trim(); + } + + + private String buildQueryString(BasicQuery query) { + + return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; + } + + + private String buildRuleString(RuleClass ruleClass) { + + StringBuilder ruleBuilder = new StringBuilder(); + + // Use RuleType to distinguish between different rule types, if needed + ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); + + for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { + for (BasicRule rule : ruleUnit.rules()) { + // Assuming BasicRule has a method to retrieve the condition as a string + ruleBuilder.append(rule.getCode()).append("\n"); + } + } + ruleBuilder.append("end"); + + return ruleBuilder.toString(); } @@ -126,6 +216,7 @@ public class RuleFileParser { private List buildRuleClasses(List allRules) { + //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv diff --git a/redaction-service-v1/rules-management/src/main/resources/order_template.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/order_template.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java new file mode 100644 index 00000000..1996af83 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -0,0 +1,149 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RuleBuilderTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); + private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); + private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + + @Autowired + RuleBuilderService ruleBuilderService; + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import(LayoutParsingServiceProcessorConfiguration.class) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RuleBuilderTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + public void removeSystemRulesTest() { + + String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); + String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + + Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + + Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + } + + + @Test + public void mergeUserRulesUpdateTest() { + + RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); + String mergedRules = mergingResult.getMergedRules(); + RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); + ruleFileBluePrintUserRulesUpdate.getRuleClasses() + .forEach(ruleClass -> { + Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); + }); + + } + + + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getRuleClasses() + .stream() + .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) + .collect(Collectors.toList()).size() == 0; + } + + + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getImports().isEmpty(); + } + + + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getQueries().isEmpty(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java index cac2af47..2d95017d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java @@ -11,10 +11,10 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.core.io.ClassPathResource; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index b8fd95fe..2e42db62 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -22,11 +22,11 @@ import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; import lombok.SneakyThrows; @@ -235,11 +235,11 @@ class DroolsValidationServiceTest { if (droolsValidation.isCompiled()) { continue; } - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(rulesString); + RuleCompilationResult baseRuleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); - rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); - rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + rulesString = rulesString.replace(ruleCompilationResult.getImports(), baseRuleCompilationResult.getImports()); + rulesString = rulesString.replace(ruleCompilationResult.getGlobals(), baseRuleCompilationResult.getGlobals()); try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); @@ -460,9 +460,9 @@ class DroolsValidationServiceTest { end """; - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled()); + assertFalse(ruleCompilationResult.getDroolsValidation().isCompiled()); } } \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java similarity index 91% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java index f8d25b6a..fa77a92b 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,10 +27,10 @@ public class DroolsCompilationTest { @Test public void testValidateRuleSyntax() throws IOException { - URL rmURL = Resources.getResource("all_redact_manager_rules.drl"); + URL rmURL = Resources.getResource("rulesmanagement/all_redact_manager_rules.drl"); String rmRule = Resources.toString(rmURL, StandardCharsets.UTF_8); - URL dmURL = Resources.getResource("all_rules_documine.drl"); + URL dmURL = Resources.getResource("rulesmanagement/all_rules_documine.drl"); String dmRule = Resources.toString(dmURL, StandardCharsets.UTF_8); assertTrue(validateRuleSyntax(rmRule).isValid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java new file mode 100644 index 00000000..577f996a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; + +public class RuleFileBluePrintMergingTest { + + @Test + public void testBothRuleFilesCanBeMerged() { + + RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); + assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); + } + +} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java similarity index 93% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java index bea04152..67c531a4 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import java.io.File; import java.nio.file.Files; @@ -8,7 +8,7 @@ import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.migration.RuleFileMigrator; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java similarity index 94% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java index d0354ab4..bc764f7d 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -19,10 +19,10 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/addNewRulesHere.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/addNewRulesHere.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl diff --git a/redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Basf-Demo/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Basf-Demo/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_function_tests/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_function_tests/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_rule_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_rule_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_security_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_security_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/PII/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/PII/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/case_insensitive/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/case_insensitive/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/no_redaction_indicator/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/no_redaction_indicator/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/vertebrate_only/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/vertebrate_only/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl new file mode 100644 index 00000000..16ea73d2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl @@ -0,0 +1,50 @@ +package drools +//------------------------------------ Table extraction rules ------------------------------------ + +// Rule unit: TAB.0 +rule "TAB.0.0: Changed Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Changed Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +// Rule unit: TAB.6 +rule "TAB.6.1: Changed Targeted cell extraction (Experimental Stop date)" + when + $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2")) + $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList() + TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList() + $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList() + then + entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date")); + end + diff --git a/redaction-service-v1/rules-management/build.gradle.kts b/redaction-service-v1/rules-management/build.gradle.kts index 2aaea40c..b005ed7e 100644 --- a/redaction-service-v1/rules-management/build.gradle.kts +++ b/redaction-service-v1/rules-management/build.gradle.kts @@ -27,10 +27,10 @@ sourceSets { } dependencies { - implementation(project(":redaction-service-server-v1")) + implementation(project(mapOf("path" to ":redaction-service-server-v1"))) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") - + implementation(project(":redaction-service-server-v1")) implementation("com.github.javaparser:javaparser-core:3.25.3") implementation("org.drools:drools-drl-parser:8.41.0.Final") implementation("org.apache.commons:commons-csv:1.10.0") diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java index 7267bd20..6ab58cf3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java @@ -17,12 +17,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; @@ -72,7 +71,6 @@ public class Main { Set identifiers = switch (type) { case "list", "l" -> RuleIdentifier.fromListOfIdentifiersString(input); case "file", "f" -> RuleFileParser.parseRuleIdentifiersFromFile(input); - case "old", "o" -> OldRulesParser.translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(input), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid", cmd.getOptionValue("t"))); }; ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType); @@ -102,7 +100,6 @@ public class Main { String type = cmd.hasOption("t") ? cmd.getOptionValue("t") : "f"; Map> identifiersPerFile = switch (type) { case "file", "f" -> parseIdentifiersFromFiles(Path.of(cmd.getOptionValue("input"))); - case "old", "o" -> translateIdentifiersFromOldFiles(Path.of(cmd.getOptionValue("input")), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid in combination with the \"recursive\" flag", cmd.getOptionValue("t"))); }; identifiersPerFile.keySet() @@ -137,13 +134,6 @@ public class Main { } - @SneakyThrows - private static Map> translateIdentifiersFromOldFiles(Path inputDirectory, ApplicationType applicationType) { - - return RuleFileIO.streamAllRuleFilesInDirectory(inputDirectory) - .collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, applicationType))); - } - @SneakyThrows private static Map> parseIdentifiersFromFiles(Path inputDirectory) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java deleted file mode 100644 index af844a6d..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -public class RuleManagementResources { - - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_redact_manager_rules.drl"); - } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl"); - } - - - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers.txt"); - } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers_dm.txt"); - } - } - - - public static InputStream getTemplateInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("order_template.txt"); - } - - - public static InputStream getOldRulesCsvInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("old_rules_with_translations.csv"); - } - - - @SneakyThrows - public static String createTempOldRulesCsv(String formattedAsCsv) { - - File csvFile = File.createTempFile("old_rules_with_translations-", ".csv"); - try (var out = new FileOutputStream(csvFile)) { - out.write(formattedAsCsv.getBytes(StandardCharsets.UTF_8)); - } - return csvFile.toString(); - } - -} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java deleted file mode 100644 index ae0f3134..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java +++ /dev/null @@ -1,203 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; -import org.drools.drl.ast.descr.PackageDescr; -import org.drools.drl.ast.descr.RuleDescr; -import org.drools.drl.parser.DrlParser; -import org.kie.internal.builder.conf.LanguageLevelOption; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.OldRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; - -@UtilityClass -public class OldRulesParser { - - static List HEADERS = List.of("id", "old rule names", "old rule code", "translates to"); - - - @SneakyThrows - public void createSetOfTranslatedRuleIdentifiersForEachFileInTheCsv() { - - List allHeaders = List.of(); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - - } - - - public Set translateOldRulesFileToNewIdentifiers(File oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesFileToNewIdentifiers(oldRulesFile.toString(), applicationType); - } - - - public Set translateOldRulesFileToNewIdentifiers(String oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(oldRulesFile), applicationType); - } - - - public Set translateEscapedOldRulesStringToNewIdentifiers(String escapedOldRulesString, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.unescapeAndUnWrap(escapedOldRulesString), applicationType); - } - - - public Set translateOldRulesStringToNewIdentifiers(String oldRulesString, ApplicationType applicationType) { - - List oldRules = parseOldRules(oldRulesString); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - Map> translationPairs = new HashMap<>(); - for (OldRule oldRule : oldRules) { - List translatedIdentifiers = records.stream() - .filter(oldRulesCsvRecord -> oldRulesCsvRecord.code().equals(oldRule.code())) - .map(OldRulesCsvRecord::translatesTo) - .findAny() - .orElse(Collections.emptyList()); - translationPairs.put(oldRule, translatedIdentifiers); - } - boolean allTranslated = true; - for (OldRule oldRule : translationPairs.keySet()) { - if (translationPairs.get(oldRule).isEmpty()) { - allTranslated = false; - records.add(new OldRulesCsvRecord(records.size(), oldRule.code(), oldRule.name(), Collections.emptyList())); - System.out.printf( - "Rule %s has not been translated yet! %nIt has been added to the bottom of the old_rules_with_translations.csv file. To continue, translate the rule and enter its identifier in the csv, then try again!%n", - oldRule.name()); - } - } - if (!allTranslated) { - String formattedAsCsv = formatAsCsv(records); - String fileLocation = RuleManagementResources.createTempOldRulesCsv(formattedAsCsv); - System.out.printf("CSV File with updated values is located at %s%n", fileLocation); - throw new IllegalArgumentException("Non translated Rule found!"); - } - RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); - - translationPairs.forEach((key, value) -> System.out.printf("Rule %s will be translated to %s \n", - key.name(), - String.format("%s: %s", - value, - value.stream() - .map(bluePrint::findRuleByIdentifier) - .flatMap(Collection::stream) - .map(BasicRule::name) - .toList()))); - - return Stream.concat(// - Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),// - translationPairs.values() - .stream() - .flatMap(Collection::stream)) - .map(ruleIdentifier -> new RuleIdentifier(ruleIdentifier.type(), ruleIdentifier.unit(), null)) - .collect(Collectors.toSet()); - } - - - @SneakyThrows - public String formatAsCsv(List records) { - - StringWriter sw = new StringWriter(); - CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS.toArray(String[]::new)).build(); - try (CSVPrinter printer = new CSVPrinter(sw, csvFormat)) { - for (OldRulesCsvRecord record : records) { - printer.printRecord(Stream.of(record.id, RuleFileIO.escapeAndWrap(record.names), RuleFileIO.escapeAndWrap(record.code), record.translatesTo)); - } - } - return sw.toString(); - } - - - @SneakyThrows - private List parseOldRules(String oldRuleString) { - - DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); - PackageDescr packageDescr = parser.parse(false, oldRuleString); - List oldRules = new LinkedList<>(); - for (RuleDescr rule : packageDescr.getRules()) { - if (!rule.isRule()) { - continue; - } - oldRules.add(OldRule.fromRuleDescr(rule, oldRuleString)); - } - return oldRules; - } - - - @SneakyThrows - public List getOldRulesCsvRecords(InputStream allRulesCsvInputStream) { - - List parsedRecords = new LinkedList<>(); - try (Reader reader = new InputStreamReader(allRulesCsvInputStream)) { - Iterable records = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord().parse(reader); - for (CSVRecord record : records) { - String[] values = record.values(); - parsedRecords.add(new OldRulesCsvRecord(Long.parseLong(values[0]), - RuleFileIO.unescapeAndUnWrap(values[2]), - RuleFileIO.unescapeAndUnWrap(values[1]), - parseRuleIdentifiers(values[3]))); - } - } - return parsedRecords; - } - - - private List parseRuleIdentifiers(String value) { - - String cleanedValue = value; - if (cleanedValue.startsWith("[")) { - cleanedValue = cleanedValue.substring(1); - } - if (cleanedValue.endsWith("]")) { - cleanedValue = cleanedValue.substring(0, cleanedValue.length() - 1); - } - if (cleanedValue.isEmpty() || cleanedValue.isBlank()) { - return Collections.emptyList(); - } - List ruleIdentifiers = new LinkedList<>(); - for (String identifier : cleanedValue.split(", ")) { - ruleIdentifiers.add(RuleIdentifier.fromString(identifier)); - } - return ruleIdentifiers; - } - - - private List parseBooleanList(String[] values) { - - return Arrays.stream(values) - .map(Boolean::parseBoolean) - .toList(); - } - - - public record OldRulesCsvRecord(long id, String code, String names, List translatesTo) { - - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java deleted file mode 100644 index 559101ac..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; - -public class RuleFileBluePrintMergingTest { - - @Test - public void testBothRuleFilesCanBeMerged() { - - RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); - assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java deleted file mode 100644 index b78ab534..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -class OldRulesParserTest { - - @Test - @SneakyThrows - @Disabled - public void translateOldRulesToNewIdentifiersTest() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("efsa_regulation_rules.txt")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateEscapedOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @SneakyThrows - @Disabled - public void translateMoreOldRules() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("basf/demo/table_demo.drl")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @Disabled - public void printTranslationsTest() { - - List records = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")) - .forEach(type -> { - List rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type) - .orElseThrow().ruleUnits(); - rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records)); - }); - } - - - private void printOldRulesThatTranslatesToNewRule(RuleUnit ruleUnit, List records) { - - if (ruleUnit.rules().isEmpty()) { - System.out.println("Rule unit empty, skipping!\n"); - return; - } - RuleIdentifier identifier = ruleUnit.rules() - .get(0).identifier(); - RuleIdentifier unitIdentifier = new RuleIdentifier(identifier.type(), identifier.unit(), null); - String oldNames = records.stream() - .filter(r -> r.translatesTo() - .stream() - .anyMatch(i -> i.matches(unitIdentifier))) - .map(OldRulesParser.OldRulesCsvRecord::names) - .map(OldRulesParserTest::removeIdFromName) - .distinct() - .collect(Collectors.joining(", ")); - System.out.println(unitIdentifier.toRuleUnitString() + " " + ruleUnit.rules() - .stream() - .map(BasicRule::name) - .collect(Collectors.joining(", "))); - System.out.println("translate from"); - System.out.println(oldNames); - System.out.println(); - System.out.println(); - } - - - private static String removeIdFromName(String name) { - - String[] values = name.split(":"); - return String.join(":", Arrays.copyOfRange(values, 1, values.length)); - } - - - @Test - @Disabled - @SneakyThrows - public void findAllCustomerRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("pilot/EFSA_sanitisation_GFL_v1").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.RM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.RM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void findAllRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("business-logic/Syngenta_RSS").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.DM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.DM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void translateDossierTemplatesV2() { - -// String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/RED/dossier-templates-v2/"; -// Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), Files.walk(Path.of(dossierTemplatesRepo + "docu")), Files.walk(Path.of(dossierTemplatesRepo + "qa"))) - String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/"; - Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), - Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) - // - ) - .flatMap(Function.identity())// - .filter(path -> path.getFileName().toString().equals("rules.drl"))// - .map(Path::toFile)// - .forEach(this::translateOldRuleFile); - - } - - - @SneakyThrows - private void translateOldRuleFile(File oldRulesFile) { - - Set identifiers = OldRulesParser.translateOldRulesFileToNewIdentifiers(oldRulesFile, ApplicationType.RM); - String newRulesString = RuleFileFactory.createFileFromIdentifiers(identifiers, ApplicationType.RM); - String result = RuleFileIO.escapeAndWrap(newRulesString); - - try (var out = new FileOutputStream(oldRulesFile)) { - out.write(result.getBytes(StandardCharsets.UTF_8)); - } - } - -} \ No newline at end of file -- 2.47.2 From 301dc718e8ffd66ddcb9fc153572ff1339e54d07 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 09:44:51 +0100 Subject: [PATCH 002/108] RED-9472: seperation of system rules code review changes --- .../controller/RuleBuilderController.java | 9 +- .../model/drools/RuleCompilationResult.java | 46 +---- .../RuleManagementResources.java | 19 +- .../factory/RuleFileParser.java | 15 +- .../models/RuleFileBluePrint.java | 28 +++ .../v1/server/service/RuleBuilderService.java | 91 ++------- .../drools/RuleCompilationResultParser.java | 83 +------- .../redaction/v1/server/RuleBuilderTest.java | 183 ++++++++---------- 8 files changed, 147 insertions(+), 327 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 8f3a51dd..aa215fc7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -45,7 +45,7 @@ public class RuleBuilderController implements RuleBuilderResource { RulesResponse rulesResponse = new RulesResponse(); String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); if (filteredRules.isEmpty()) { - throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + throw new AssertionError("There was an error when cleaning the rulefile of sytem rules"); } rulesResponse.setRules(filteredRules); return rulesResponse; @@ -53,13 +53,13 @@ public class RuleBuilderController implements RuleBuilderResource { @Override - public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); DroolsValidationResponse droolsValidationResponse; RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); if (mergingResult.getMergedRules().isEmpty()) { - throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); @@ -90,7 +90,8 @@ public class RuleBuilderController implements RuleBuilderResource { .toList()) .build(); if (!droolsValidation.isCompiled()) { - return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.UNPROCESSABLE_ENTITY); } else { // rulesUploadResponse.setRules(mergingResult.getMergedRules()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index c7e35460..fe595301 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -47,60 +47,16 @@ public final class RuleCompilationResult { public void dropRulesByIdentifier(RuleType ruleType) { - log.info("removing rule with identifier {}", ruleType.name()); - List rulesToBeRemoved = ruleClasses.stream() .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); ruleClasses.removeAll(rulesToBeRemoved); } - public void dropAllRulesExceptSystemRules(List systemRules) { - - log.info("removing all rules except the system rules"); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) - .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); - ruleClasses.removeAll(rulesToBeRemoved); - } - - - public void dropImports() { - - this.imports = ""; - } - - - public void dropQueries() { - - this.queries.clear(); - } - - - public int countRuleOccurences(RuleType ruleType) { - - log.info("counting occurences of files {}", ruleType.name()); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) - .toList(); - return rulesToBeRemoved.size(); - } - - - private boolean filterOutRule(RuleType ruleType, List filteredRules) { - - return !filteredRules.contains(ruleType.name()); - } - - public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replace("\n", "").split("import")) + return Arrays.stream(imports.replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java index 331b1e25..e423dde8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -1,34 +1,37 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement; +import java.io.IOException; import java.io.InputStream; +import org.drools.io.ClassPathResource; + import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; @SuppressWarnings("PMD") public class RuleManagementResources { - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + public static InputStream getAllRulesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + return new ClassPathResource("drools/all_redact_manager_rules.drl").getInputStream(); } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + return new ClassPathResource("drools/all_rules_documine.drl").getInputStream(); } - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers.txt").getInputStream(); } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers_dm.txt").getInputStream(); } } - public static InputStream getTemplateInputStream() { + public static InputStream getTemplateInputStream() throws IOException { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + return new ClassPathResource("rulesmanagement/order_template.txt").getInputStream(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index db0ece77..6354f93c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -4,7 +4,6 @@ import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -121,16 +120,12 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - if (rulesPerUnit.keySet() != null) { - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); - } else { - return new ArrayList<>(); - } + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 28b2663c..ca185b63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; @@ -53,6 +54,33 @@ public class RuleFileBluePrint { } + public boolean removeRuleClassByRuleIdentifier(RuleIdentifier ruleIdentifier) { + + AtomicBoolean wasRemoved = new AtomicBoolean(false); + + findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> { + List unitsToRemove = new ArrayList<>(); + + ruleClass.ruleUnits() + .forEach(ruleUnit -> { + boolean removed = ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier)); + if (removed) { + wasRemoved.set(true); + if (ruleUnit.rules().isEmpty()) { + unitsToRemove.add(ruleUnit); + } + } + }); + + ruleClass.ruleUnits().removeAll(unitsToRemove); + if (ruleClass.ruleUnits().isEmpty()) { + this.ruleClasses.remove(ruleClass); + } + }); + return wasRemoved.get(); + } + + public Optional findRuleClassByType(RuleType ruleType) { return ruleClasses.stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 32aaf4fc..d20e8337 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -3,9 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -16,8 +15,6 @@ import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFil import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.extern.slf4j.Slf4j; @@ -25,13 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - //todo: make this configurable - private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), - new RuleType("MAN"), - new RuleType("X"), - new RuleType("DICT"), - new RuleType("FA"), - new RuleType("LDS"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); public RuleBuilderModel getRuleBuilderModel() { @@ -48,28 +44,24 @@ public class RuleBuilderService { public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - log.info("Starting to remove system rules from ruleFile"); - removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); - log.info("Finished removing system rules for ruleFile"); - + removeSystemRules(ruleFileBluePrint); return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); } public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { - log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() - .map(RuleType::name) - .toList().contains(ruleClass.ruleType().name())) { - throw new RuntimeException("No system rule updates allowed in user rule update."); + .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { + log.warn("System rule in user rule update."); + //throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); ruleFileBluePrintExisting.getRuleClasses() .stream() .flatMap(ruleClass -> ruleClass.ruleUnits() @@ -89,7 +81,6 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - log.info("finished merging user rules update with system rules"); RuleMergingResult mergingResult = RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) @@ -100,69 +91,17 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { - - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); - removeAllRulesExceptSystemRules(ruleFileBluePrint); - ruleFileBluePrint.getRuleClasses() - .forEach(ruleClass -> { - if (!systemRuleNames.contains(ruleClass.ruleType().name())) { - throw new RuntimeException("there was an error removing all rules except system rules"); - } - }); - } - - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - List rules = new ArrayList(); - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); - ruleFileBluePrint.getRuleClasses() - .stream() - .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) - .forEach(rule -> rules.add(rule)); - rules.forEach(ruleIdentifier -> { - ruleFileBluePrint.removeRule(ruleIdentifier); - }); - } - - - private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { - - removeSystemRules(ruleFileBluePrint); - for (RuleType systemRule : systemRules) { - List remainingSystemRules = new ArrayList(); - ruleFileBluePrint.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(remainingSystemRules::add)); - if (!remainingSystemRules.isEmpty()) { - throw new RuntimeException("There was an error removing the system rules from the file"); - } - } } private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { - for (RuleType systemRule : systemRules) { - List rules = new ArrayList(); - ruleFileBluePrintExisting.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(rules::add)); - rules.forEach(ruleUnit -> { - ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); - }); - ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + for (RuleIdentifier systemRule : systemRules) { + ruleFileBluePrintExisting.removeRuleClassByRuleIdentifier(systemRule); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 2a5158f3..1995b69e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -35,33 +35,6 @@ public class RuleCompilationResultParser { "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { - - DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); - List allRules = new LinkedList<>(); - List allQueries = new LinkedList<>(); - PackageDescr packageDescr = new PackageDescr(); - String imports = ""; - String globals = ""; - List ruleClasses = buildRuleClasses(allRules); - return new RuleCompilationResult(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); - } - - @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { @@ -116,60 +89,7 @@ public class RuleCompilationResultParser { @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { - return buildRuleCompilationResultFromRuleString(ruleString,false); - } - - - @SneakyThrows - public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { - - StringBuilder ruleStringBuilder = new StringBuilder(); - - // Append imports - ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); - - // Append globals - if (!bluePrint.getGlobals().isEmpty()) { - ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); - } - - // Append queries - for (BasicQuery query : bluePrint.getQueries()) { - ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); - } - - // Append rules - for (RuleClass ruleClass : bluePrint.getRuleClasses()) { - ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); - } - - // Return the final rule string - return ruleStringBuilder.toString().trim(); - } - - - private String buildQueryString(BasicQuery query) { - - return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; - } - - - private String buildRuleString(RuleClass ruleClass) { - - StringBuilder ruleBuilder = new StringBuilder(); - - // Use RuleType to distinguish between different rule types, if needed - ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); - - for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { - for (BasicRule rule : ruleUnit.rules()) { - // Assuming BasicRule has a method to retrieve the condition as a string - ruleBuilder.append(rule.getCode()).append("\n"); - } - } - ruleBuilder.append("end"); - - return ruleBuilder.toString(); + return buildRuleCompilationResultFromRuleString(ruleString, false); } @@ -216,7 +136,6 @@ public class RuleCompilationResultParser { private List buildRuleClasses(List allRules) { - //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 1996af83..181e108a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,149 +1,128 @@ package com.iqser.red.service.redaction.v1.server; +import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; import static org.mockito.Mockito.when; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; -import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; -import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class RuleBuilderTest extends AbstractRedactionIntegrationTest { +import org.junit.jupiter.api.Assertions; +import lombok.SneakyThrows; - private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); - private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); - private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); - - @Autowired - RuleBuilderService ruleBuilderService; - - @Configuration - @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) - @Import(LayoutParsingServiceProcessorConfiguration.class) - @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) - public static class RuleBuilderTestConfiguration { - - @Bean - @Primary - public StorageService inmemoryStorage() { - - return new FileSystemBackedStorageService(ObjectMapperFactory.create()); - } - - } +class RuleBuilderTest { + private RuleBuilderService ruleBuilderService; + private final List systemRules = new ArrayList<>(Arrays.asList( + RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*") + )); @BeforeEach - public void stubClients() { - + void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); - - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); - - loadDictionaryForTest(); - loadTypeForTest(); - loadNerForTest(); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); - - when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); - - mockDictionaryCalls(null); - - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } - @Test - public void removeSystemRulesTest() { + @SneakyThrows + void removeSystemRulesTest() throws IOException { + URL url = getClass().getClassLoader().getResource("drools"); + if (url == null) { + throw new RuntimeException("Drools directory not found in resources"); + } - String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); - String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); - RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + Path path = Paths.get(url.toURI()); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); - Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); - - Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } + private void testThatSystemRulesAreRemoved(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + true + ); - @Test - public void mergeUserRulesUpdateTest() { + RuleCompilationResult ruleCompilationResultWithImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); - RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); - String mergedRules = mergingResult.getMergedRules(); - RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); - ruleFileBluePrintUserRulesUpdate.getRuleClasses() - .forEach(ruleClass -> { - Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); - }); + Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports + ); + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + + Assertions.assertEquals( + RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + } private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getRuleClasses() - .stream() - .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) - .collect(Collectors.toList()).size() == 0; + .stream() + .map(RuleClass::ruleType) + .noneMatch(ruleType -> systemRules.stream() + .map(RuleIdentifier::type) + .anyMatch(type -> ruleType.name().equals(type.name()))); } - private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getImports().isEmpty(); } - private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getQueries().isEmpty(); } - -} +} \ No newline at end of file -- 2.47.2 From 65166751f1f65aa36cce8f6f50fab9bfa70a6107 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 003/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 1995b69e..73392fa8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 5ab0747d303e85ccf9b389c385348df6ae306807 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 004/108] RED-9472: seperation of system rules pmd fix --- .../v1/server/service/RuleBuilderService.java | 20 +++++++++---------- .../drools/RuleCompilationResultParser.java | 8 ++------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index d20e8337..2c9c9f7e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -58,7 +58,7 @@ public class RuleBuilderService { if (systemRules.stream() .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { log.warn("System rule in user rule update."); - //throw new RuntimeException("No system rule updates allowed in user rule update."); + throw new RuntimeException("No system rule updates allowed in user rule update."); } }); removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); @@ -81,19 +81,17 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - RuleMergingResult mergingResult = RuleMergingResult.builder() + return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) .build(); - - return mergingResult; } private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 73392fa8..1995b69e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 07cd81e5a9cda1f389444d2cbaa09f907b46a8a9 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:26:32 +0100 Subject: [PATCH 005/108] RED-9472: seperation of system rules pmd fix --- .../redaction/v1/server/RuleBuilderTest.java | 83 ++++++++----------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 181e108a..31d61e1e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server; -import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URL; import java.nio.file.Path; @@ -13,8 +8,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; @@ -27,35 +24,32 @@ import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.knecon.fforesight.tenantcommons.TenantContext; -import org.junit.jupiter.api.Assertions; import lombok.SneakyThrows; class RuleBuilderTest { private RuleBuilderService ruleBuilderService; - private final List systemRules = new ArrayList<>(Arrays.asList( - RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*") - )); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); + @BeforeEach void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); } + @Test @SneakyThrows void removeSystemRulesTest() throws IOException { - URL url = getClass().getClassLoader().getResource("drools"); - if (url == null) { - throw new RuntimeException("Drools directory not found in resources"); - } + URL url = Thread.currentThread().getContextClassLoader().getResource("drools"); Path path = Paths.get(url.toURI()); RuleFileIO.streamAllRuleFilesInDirectory(path) .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); @@ -64,20 +58,14 @@ class RuleBuilderTest { .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } - private void testThatSystemRulesAreRemoved(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - true - ); - RuleCompilationResult ruleCompilationResultWithImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + private void testThatSystemRulesAreRemoved(Path path) { + + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), true); + + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); @@ -88,28 +76,22 @@ class RuleBuilderTest { Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - cleanedRulesWithImports - ); + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports); - Assertions.assertEquals( - RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(cleanedRuleFileMergedWithSystemRules.getMergedRules()); + + Assertions.assertEquals(RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), cleanedRuleFileMergedWithSystemRules.getMergedRules()); } + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getRuleClasses() .stream() .map(RuleClass::ruleType) @@ -118,11 +100,16 @@ class RuleBuilderTest { .anyMatch(type -> ruleType.name().equals(type.name()))); } + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getImports().isEmpty(); } + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getQueries().isEmpty(); } + } \ No newline at end of file -- 2.47.2 From 7e64bb28e880b7006d0dfcdd83896b685b343321 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 15 Nov 2024 09:45:56 +0100 Subject: [PATCH 006/108] RED-9472: seperation of system rules review changes --- .../models/RuleFileBluePrint.java | 2 +- .../v1/server/service/RuleBuilderService.java | 46 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index ca185b63..533492e1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -180,7 +180,7 @@ public class RuleFileBluePrint { public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), declarations, functions); ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 2c9c9f7e..c346429c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -1,10 +1,10 @@ package com.iqser.red.service.redaction.v1.server.service; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), - RuleIdentifier.fromName("MAN"), - RuleIdentifier.fromName("X"), - RuleIdentifier.fromName("DICT"), - RuleIdentifier.fromName("FA"), - RuleIdentifier.fromName("LDS"))); + private final HashSet systemRules = new HashSet<>(Set.of(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -52,7 +52,7 @@ public class RuleBuilderService { public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() @@ -61,25 +61,30 @@ public class RuleBuilderService { throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - ruleFileBluePrintExisting.getRuleClasses() - .stream() - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .flatMap(ruleUnit -> ruleUnit.rules() - .stream())) + ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getAllRules() .forEach(mergedRuleFileBlueprint::addRule); - mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); - mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setImports(uniqueImports.stream() + .collect(Collectors.joining(""))); + Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setGlobals(uniqueGlobals.stream() + .collect(Collectors.joining(""))); mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() .stream(), mergedRuleFileBlueprint.getFunctions() .stream()) + .distinct() .toList()); mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() .stream(), mergedRuleFileBlueprint.getDeclarations() .stream()) + .distinct() .toList()); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) @@ -89,10 +94,9 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + private RuleFileBluePrint removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + return ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } -- 2.47.2 From 85bef908a08ec8d6a69ed3a5de15cb85625c3e1c Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Sep 2024 10:24:59 +0200 Subject: [PATCH 007/108] RED-9472: seperation of system rules added endpoints for seperating and readding system rules working on service RED-9472: seperation of system rules refactored endpoints. now removing system rules on download merging user rules with systemrules on upload RED-9472: seperation of system rules RED-9472: seperation of system rules RED-9472: seperation of system rules refactored two have a rulefileblueprint and a rulefileblueprint for validationresult changed the flow of merging and seperating system and user rules added functions and declarations to rule file parsing RED-9472: seperation of system rules RED-9472: seperation of system rules introduced tests working on fixing bug with merging rules RED-9472: seperation of system rules introduced tests RED-9472: seperation of system rules fixed circular depdency RED-9472: seperation of system rules updated persistence service version RED-9472: seperation of system rules refactored rules refactored pcackages RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed some sonarqube errors RED-9472: seperation of system rules fixed pmd error RED-9472: seperation of system rules added deploy stage to gitlab pipeline RED-9472: seperation of system rules RED-9472: seperation of system rules fixed a mistake RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse --- .../redaction-service-api-v1/build.gradle.kts | 3 +- .../v1/resources/RuleBuilderResource.java | 17 +- .../build.gradle.kts | 32 ++- .../controller/RuleBuilderController.java | 96 ++++++++- .../v1/server/model/RuleMergingResult.java | 24 +++ ...ePrint.java => RuleCompilationResult.java} | 67 +++++- .../RuleManagementResources.java | 34 +++ .../DefaultRuleIdentifiersFactory.java | 8 +- .../factory/RuleFileFactory.java | 51 +++-- .../factory/RuleFileParser.java | 82 +++++-- .../migration/RuleFileMigrator.java | 15 +- .../migration/RuleIdentifierMigrator.java | 45 ++-- .../models/ApplicationType.java | 2 +- .../models/BasicDeclaration.java | 34 +++ .../rulesmanagement/models/BasicFunction.java | 36 ++++ .../rulesmanagement}/models/BasicRule.java | 4 +- .../rulesmanagement}/models/OldRule.java | 2 +- .../rulesmanagement}/models/RuleClass.java | 2 +- .../models/RuleFileBluePrint.java | 42 +++- .../models/RuleIdentifier.java | 2 +- .../rulesmanagement}/models/RuleType.java | 2 +- .../rulesmanagement}/models/RuleUnit.java | 2 +- .../rulesmanagement}/utils/RuleFileIO.java | 2 +- .../v1/server/service/RuleBuilderService.java | 169 +++++++++++++++ .../drools/DroolsValidationService.java | 85 ++++---- ....java => RuleCompilationResultParser.java} | 139 +++++++++--- .../all_redact_manager_rules.drl | 0 .../rulesmanagement}/all_rules_documine.drl | 0 .../default_rule_identifiers.txt | 0 .../default_rule_identifiers_dm.txt | 0 .../old_rules_with_translations.csv | 0 .../rulesmanagement}/order_template.txt | 0 .../redaction/v1/server/RuleBuilderTest.java | 149 +++++++++++++ .../services/DroolsUpToDateTest.java | 8 +- .../services/DroolsValidationServiceTest.java | 16 +- .../DroolsCompilationTest.java | 6 +- .../RuleFileBluePrintMergingTest.java | 20 ++ .../RuleFileMigrationTest.java | 4 +- .../factory/RuleFileFactoryTest.java | 10 +- .../EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rulesmanagement}/addNewRulesHere.drl | 0 .../rulesmanagement}/demo/table_demo.drl | 0 .../rulesmanagement}/dev/Basf-Demo/rules.txt | 0 .../rulesmanagement}/dev/Flora/rules.txt | 0 .../dev/EFSA_Regulation_2021/rules.txt | 0 .../dev/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rules.txt | 0 .../EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../dev/Manual_Redaction/rules.txt | 0 .../docu/EFSA_Regulation_2021/rules.txt | 0 .../docu/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../docu/Manual_Redaction/rules.txt | 0 .../qa/EFSA_Regulation_2021/rules.txt | 0 .../qa/EFSA_Regulation_2022/rules.txt | 0 .../qa/EFSA_acceptance_test/rules.txt | 0 .../qa/EFSA_function_tests/rules.txt | 0 .../qa/EFSA_rule_test/rules.txt | 0 .../qa/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rules.txt | 0 .../qa/EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../qa/EFSA_security_test/rules.txt | 0 .../qa/EFSA_smoke_tests/rules.txt | 0 .../qa/Manual_Redaction/rules.txt | 0 .../dossier_templates_v2/qa/PII/rules.txt | 0 .../qa/case_insensitive/rules.txt | 0 .../qa/no_redaction_indicator/rules.txt | 0 .../qa/vertebrate_only/rules.txt | 0 .../efsa_regulation_rules.txt | 0 .../userrules/user_rule_update.drl | 50 +++++ .../rules-management/build.gradle.kts | 4 +- .../utility/rules/management/Main.java | 20 +- .../management/RuleManagementResources.java | 56 ----- .../translation/OldRulesParser.java | 203 ------------------ .../RuleFileBluePrintMergingTest.java | 20 -- .../translation/OldRulesParserTest.java | 176 --------------- 76 files changed, 1065 insertions(+), 674 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/{RuleFileBluePrint.java => RuleCompilationResult.java} (61%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/DefaultRuleIdentifiersFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileParser.java (50%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleFileMigrator.java (77%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleIdentifierMigrator.java (76%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/ApplicationType.java (73%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/BasicRule.java (80%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/OldRule.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleClass.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleFileBluePrint.java (85%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleIdentifier.java (97%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleType.java (95%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleUnit.java (54%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/utils/RuleFileIO.java (96%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/{RuleFileParser.java => RuleCompilationResultParser.java} (53%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_redact_manager_rules.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_rules_documine.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers_dm.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/old_rules_with_translations.csv (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/order_template.txt (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/DroolsCompilationTest.java (91%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/RuleFileMigrationTest.java (93%) rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactoryTest.java (94%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/addNewRulesHere.drl (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/demo/table_demo.drl (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dev/Basf-Demo/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dev/Flora/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_function_tests/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_rule_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_security_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/PII/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/case_insensitive/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/no_redaction_indicator/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/vertebrate_only/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/efsa_regulation_rules.txt (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index a4790e05..ccff795e 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,11 +4,12 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.612.0-RED10072.1" +val persistenceServiceVersion = "2.585.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + api("com.knecon.fforesight:swagger-commons:0.7.0") } publishing { diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index 3864ba9b..4683fd5f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,13 +1,26 @@ package com.iqser.red.service.redaction.v1.resources; -import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; - import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; public interface RuleBuilderResource { @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE) RuleBuilderModel getRuleBuilderModel(); + + @PostMapping(value = "/internal-api/rules/user-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithoutSystemRules(@RequestBody SystemRulesSeperationRequest systemRulesSeperationRequest); + + + @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + ResponseEntity mergeUserUpdateRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); + } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 2d67fb11..277f9fde 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -12,12 +12,12 @@ plugins { description = "redaction-service-server-v1" -val layoutParserVersion = "0.191.0" +val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.612.0-RED10072.1" -val llmServiceVersion = "1.20.0-RED10072.2" +val persistenceServiceVersion = "2.585.0-RED9472.1" +val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" @@ -31,10 +31,15 @@ configurations { } } +configurations.all { + resolutionStrategy { + force("com.google.protobuf:protobuf-java:4.27.1") + } +} + dependencies { implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } - implementation(project(":document")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { @@ -47,7 +52,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.50.0") - implementation("com.knecon.fforesight:tenant-commons:0.31.0") + implementation("com.knecon.fforesight:tenant-commons:0.30.0") implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } @@ -56,14 +61,13 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") - implementation("org.ahocorasick:ahocorasick:0.9.0") - implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") - implementation("com.github.roklenarcic:aho-corasick:1.2") + implementation("org.ahocorasick:ahocorasick:0.6.3") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") implementation("org.drools:drools-mvel:${droolsVersion}") implementation("org.kie:kie-spring:7.74.1.Final") + implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.locationtech.jts:jts-core:1.19.0") @@ -79,6 +83,7 @@ dependencies { implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") + api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") @@ -125,7 +130,6 @@ tasks.named("bootBuildImage") { "BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5" ) - environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE! if (project.hasProperty("buildbootDockerHostNetwork")) { @@ -180,19 +184,13 @@ tasks.register("generateJavaDoc", Javadoc::class) { dependsOn("compileJava") dependsOn("delombok") classpath = project.sourceSets["main"].runtimeClasspath - val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { + source = fileTree("${buildDir}/generated/sources/delombok/java/main") { include(droolsImports) } - val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { - include(droolsImports) - } - source = documentFiles + mainFiles - - setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "")) + destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "") options.memberLevel = JavadocMemberLevel.PUBLIC (options as StandardJavadocDocletOptions).apply { title = "API Documentation for Redaction Service ${project.version}" } } - diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 306dccbe..8f3a51dd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,27 +1,111 @@ package com.iqser.red.service.redaction.v1.server.controller; -import java.util.Collections; - +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleBlacklistErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxWarningMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @RestController +@Slf4j @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { + private final RuleBuilderService ruleBuilderService; + private final DroolsValidationService droolsValidationService; + + @Override public RuleBuilderModel getRuleBuilderModel() { - RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + return this.ruleBuilderService.getRuleBuilderModel(); + } - ruleBuilderModel.setWhenClauses(Collections.emptyList()); - ruleBuilderModel.setThenConditions(Collections.emptyList()); - return ruleBuilderModel; + @Override + public RulesResponse getRuleFileWithoutSystemRules(SystemRulesSeperationRequest systemRulesSeperationRequest) { + + RulesResponse rulesResponse = new RulesResponse(); + String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); + if (filteredRules.isEmpty()) { + throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + } + rulesResponse.setRules(filteredRules); + return rulesResponse; + } + + + @Override + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + + RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); + DroolsValidationResponse droolsValidationResponse; + RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + if (mergingResult.getMergedRules().isEmpty()) { + throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + } + try { + var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + droolsValidationResponse = DroolsValidationResponse.builder() + .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() + .stream() + .map(droolsSyntaxErrorMessage -> new RuleSyntaxErrorMessage(droolsSyntaxErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxErrorMessage.getColumn(), + droolsSyntaxErrorMessage.getMessage())) + .toList()) + .deprecatedWarnings(droolsValidation.getDeprecatedWarnings() + .stream() + .map(droolsSyntaxDeprecatedWarnings -> new RuleSyntaxWarningMessage(droolsSyntaxDeprecatedWarnings.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxDeprecatedWarnings.getColumn(), + droolsSyntaxDeprecatedWarnings.getMessage())) + .toList()) + .blacklistErrorMessages(droolsValidation.getBlacklistErrorMessages() + .stream() + .map(droolsBlacklistErrorMessage -> new RuleBlacklistErrorMessage(droolsBlacklistErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsBlacklistErrorMessage.getColumn(), + droolsBlacklistErrorMessage.getMessage())) + .toList()) + .build(); + if (!droolsValidation.isCompiled()) { + return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + } else { + // + rulesUploadResponse.setRules(mergingResult.getMergedRules()); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + } + } catch (Exception e) { + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + } + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); + } + + + private enum RuleFileType { + ENTITY, + COMPONENT } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java new file mode 100644 index 00000000..b2cb1b27 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Schema(description = "Object containing a string of Drools rules.") +public class RuleMergingResult { + + @Schema(description = "The merged rules.") + private String mergedRules; + + @Schema(description = "the length of added imports from sytemRules") + private int addedImportsOffset; + @Schema(description = "the length of added globals from sytemRules") + private int addedGlobalsOffset; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java similarity index 61% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index 3aa1cdef..e96aa697 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -20,13 +20,15 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Builder @NoArgsConstructor @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public final class RuleFileBluePrint { +public final class RuleCompilationResult { String imports; int importLine; @@ -45,9 +47,62 @@ public final class RuleFileBluePrint { } + public void dropRulesByIdentifier(RuleType ruleType) { + + log.info("removing rule with identifier {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropAllRulesExceptSystemRules(List systemRules) { + + log.info("removing all rules except the system rules"); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropImports() { + + this.imports = ""; + } + + + public void dropQueries() { + + this.queries.clear(); + } + + + public int countRuleOccurences(RuleType ruleType) { + + log.info("counting occurences of files {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + return rulesToBeRemoved.size(); + } + + + private boolean filterOutRule(RuleType ruleType, List filteredRules) { + + return !filteredRules.contains(ruleType.name()); + } + + public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replaceAll("\n", "").split("import")) + return Arrays.stream(imports.replace("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } @@ -76,7 +131,7 @@ public final class RuleFileBluePrint { public List getAllRuleIdentifiers() { return streamAllRules().map(BasicRule::getIdentifier) - .collect(Collectors.toList()); + .toList(); } @@ -96,4 +151,10 @@ public final class RuleFileBluePrint { return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']'; } + + public void addRuleClass(RuleClass ruleClass) { + + this.ruleClasses.add(ruleClass); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java new file mode 100644 index 00000000..331b1e25 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import java.io.InputStream; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; + +@SuppressWarnings("PMD") +public class RuleManagementResources { + + public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + } + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + } + + + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + } else { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + } + } + + + public static InputStream getTemplateInputStream() { + + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java index 2a8de641..c83f7f04 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java @@ -1,12 +1,12 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java index 039664bf..f6332332 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.Collections.emptySet; @@ -9,13 +9,13 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -48,17 +48,17 @@ public class RuleFileFactory { } RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers); - return buildRuleString(filteredBluePrint); + return buildRuleString(filteredBluePrint, false, false); } @SneakyThrows - public String buildRuleString(RuleFileBluePrint bluePrint) { + public String buildRuleString(RuleFileBluePrint bluePrint, boolean dropImports, boolean dropQueries) { try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) { String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8); List templateRuleOrder = parseRuleOrder(template); - return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder); + return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder, dropImports, dropQueries); } } @@ -72,23 +72,35 @@ public class RuleFileFactory { } - private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder) { + private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder, boolean dropImports, boolean dropQueries) { - Set additionalRuleTypes = bluePrint.ruleClasses() + Set additionalRuleTypes = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleType) .filter(ruleType -> !ruleOrder.contains(ruleType)) .collect(Collectors.toSet()); StringBuilder sb = new StringBuilder(); - sb.append(bluePrint.imports()); + if (!dropImports) { + sb.append(bluePrint.getImports()); + sb.append("\n\n"); + } else { + //todo: this is hacked to enable compiling the rules without imports + sb.append("package drools"); + sb.append("\n\n"); + } + sb.append(bluePrint.getGlobals()); sb.append("\n\n"); - sb.append(bluePrint.globals()); + sb.append("//------------------------------------ declarations ------------------------------------"); sb.append("\n\n"); - sb.append("//------------------------------------ queries ------------------------------------"); - sb.append("\n\n"); - sb.append(bluePrint.queries()); + sb.append(bluePrint.getDeclarations().isEmpty() ? "" : bluePrint.getDeclarations()); sb.append("\n\n"); + if (!dropQueries) { + sb.append("//------------------------------------ queries ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getQueries()); + sb.append("\n\n"); + } for (RuleType ruleBlockType : ruleOrder) { if (ruleBlockType.isWildCard()) { additionalRuleTypes.stream() @@ -101,6 +113,9 @@ public class RuleFileFactory { } writeRuleClass(bluePrint, ruleBlockType, sb); } + sb.append("//------------------------------------ functions ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getFunctions().isEmpty() ? "" : bluePrint.getFunctions()); return sb.toString().trim() + "\n"; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java similarity index 50% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index 3e17b0db..db0ece77 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -1,30 +1,35 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; +import org.drools.drl.ast.descr.FunctionDescr; import org.drools.drl.ast.descr.ImportDescr; import org.drools.drl.ast.descr.PackageDescr; import org.drools.drl.ast.descr.RuleDescr; import org.drools.drl.parser.DrlParser; import org.kie.internal.builder.conf.LanguageLevelOption; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicDeclaration; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicFunction; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -45,10 +50,19 @@ public class RuleFileParser { @SneakyThrows public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString) { + return buildBluePrintFromRulesString(rulesString, false); + } + + + @SneakyThrows + public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString, boolean removeImports) { + DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); PackageDescr packageDescr = parser.parse(false, rulesString); StringBuilder queryBuilder = new StringBuilder(); List allRules = new LinkedList<>(); + List functions = new LinkedList<>(); + List declarations = new LinkedList<>(); for (RuleDescr rule : packageDescr.getRules()) { if (rule.isQuery()) { queryBuilder.append(rulesString, rule.getStartCharacter(), rule.getEndCharacter()); @@ -57,12 +71,30 @@ public class RuleFileParser { } allRules.add(BasicRule.fromRuleDescr(rule, rulesString)); } - String imports = rulesString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + for (FunctionDescr function : packageDescr.getFunctions()) { + functions.add(BasicFunction.fromFunctionDescr(function, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getEnumDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getClassAndEnumDeclarationDescrs()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + String imports = ""; + if (!removeImports && !packageDescr.getImports().isEmpty()) { + imports = rulesString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -70,7 +102,7 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses); + return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses, declarations, functions); } @@ -89,11 +121,17 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); + if (rulesPerUnit.keySet() != null) { + + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); + } else { + return new ArrayList<>(); + } + } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java similarity index 77% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java index c002fed3..afbdff26 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java @@ -1,15 +1,16 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.File; import java.io.FileOutputStream; import java.nio.charset.StandardCharsets; import java.util.List; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -30,7 +31,7 @@ public class RuleFileMigrator { //replaceRules(ruleFileBluePrint, combinedBluePrint); replaceRuleIdentifiers(combinedBluePrint, ruleFileBluePrint); - String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); + String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java similarity index 76% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java index fc8e9026..3889fad1 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.FileOutputStream; import java.io.IOException; @@ -9,16 +9,15 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -35,7 +34,6 @@ public class RuleIdentifierMigrator { @SneakyThrows public void migrateAllRuleIdentifiers() { - List parsedRecords = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); RuleFileBluePrint bluePrint = getBluePrint(ApplicationType.RM); // migrateIdentifier(RuleIdentifier.fromString("PII.10.0"), RuleIdentifier.fromString("CBI.20.0"), bluePrint, parsedRecords); @@ -46,20 +44,17 @@ public class RuleIdentifierMigrator { bluePrint = migrateMatchedRuleForAllRules(bluePrint); - String ruleString = RuleFileFactory.buildRuleString(bluePrint); + String ruleString = RuleFileFactory.buildRuleString(bluePrint, false, false); try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) { out.write(ruleString.getBytes(StandardCharsets.UTF_8)); } - String csvString = OldRulesParser.formatAsCsv(parsedRecords); - try (var out = new FileOutputStream("/tmp/old_rules_with_translations.csv")) { - out.write(csvString.getBytes(StandardCharsets.UTF_8)); - } } + //todo: introduce functions and declarations private static RuleFileBluePrint migrateMatchedRuleForAllRules(RuleFileBluePrint bluePrint) { - List migratedRules = bluePrint.ruleClasses() + List migratedRules = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) @@ -67,7 +62,12 @@ public class RuleIdentifierMigrator { .flatMap(Collection::stream) .map(RuleIdentifierMigrator::migrateMatchedRule) .toList(); - RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.imports(), bluePrint.globals(), bluePrint.queries(), new LinkedList<>()); + RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.getImports(), + bluePrint.getGlobals(), + bluePrint.getQueries(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>()); migratedRules.forEach(migratedBluePrint::addRule); return migratedBluePrint; } @@ -118,7 +118,7 @@ public class RuleIdentifierMigrator { } - public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List records) { + public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint) { BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()) .orElseThrow().findRuleUnitByInteger(oldIdentifier.unit()) @@ -132,9 +132,6 @@ public class RuleIdentifierMigrator { bluePrint.removeRule(oldIdentifier); bluePrint.addRule(newRule); - records.stream() - .filter(record -> record.translatesTo().contains(oldIdentifier)) - .forEach(record -> replaceOldIdentifier(oldIdentifier, newIdentifier, record.translatesTo())); } @@ -152,7 +149,7 @@ public class RuleIdentifierMigrator { } - public record Context(RuleFileBluePrint bluePrint, List records) { + public record Context(RuleFileBluePrint bluePrint) { } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java similarity index 73% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java index 8fdac0d7..53d1122c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import lombok.Getter; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java new file mode 100644 index 00000000..4e3acb5c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicDeclaration { + + @EqualsAndHashCode.Include + String declarationFullTypeName; + String body; + int line; + + + public static BasicDeclaration fromDeclarationDescription(AbstractClassTypeDeclarationDescr declaration, String rulesString) { + + String declarationFullTypeName = declaration.getFullTypeName(); + String body = rulesString.substring(declaration.getStartCharacter(), declaration.getEndCharacter()); + int line = declaration.getLine(); + + return new BasicDeclaration(declarationFullTypeName, body, line); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java new file mode 100644 index 00000000..1793d589 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java @@ -0,0 +1,36 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.FunctionDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicFunction { + + @EqualsAndHashCode.Include + String functionName; + String returnType; + String body; + int line; + + + public static BasicFunction fromFunctionDescr(FunctionDescr function, String rulesString) { + + String functionName = function.getName(); + String returnType = function.getReturnType(); + String body = rulesString.substring(function.getStartCharacter(), function.getEndCharacter()); + int line = function.getLine(); + + return new BasicFunction(functionName, returnType, body, line); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java similarity index 80% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java index 9d77bbb6..cc5c0fe6 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java @@ -1,8 +1,8 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; public record BasicRule(RuleIdentifier identifier, String name, String code) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java index a6cdb0ab..bb0e2d79 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java index 40468cc0..e70acc0f 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.LinkedList; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java similarity index 85% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 2b2ae91f..28b2663c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Collection; import java.util.Collections; @@ -10,7 +10,29 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -public record RuleFileBluePrint(String imports, String globals, String queries, List ruleClasses) { +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RuleFileBluePrint { + + String imports; + String globals; + String queries; + List ruleClasses; + List declarations; + List functions; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -24,7 +46,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, ruleClass.ruleUnits().remove(ruleUnit); } if (ruleClass.ruleUnits().isEmpty()) { - ruleClasses().remove(ruleClass); + this.ruleClasses.remove(ruleClass); } })); return wasRemoved.get(); @@ -41,7 +63,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRules() { - return ruleClasses().stream() + return this.ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -101,7 +123,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRuleIdentifiers() { - return ruleClasses().stream() + return ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -122,10 +144,16 @@ public record RuleFileBluePrint(String imports, String globals, String queries, } + public void setImports(String newImports) { + + this.imports = newImports; + } + + public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>()); - ruleClasses().stream() + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) .flatMap(ruleUnit -> ruleUnit.rules() diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java similarity index 97% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java index 569412e4..3fce64a7 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Arrays; import java.util.Objects; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java similarity index 95% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java index ffddc766..fcacf44d 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Map; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java similarity index 54% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java index ec23ad27..a2b84ba3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java similarity index 96% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java index 3d3ccac6..8efecd93 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.utils; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.utils; import java.io.File; import java.io.FileInputStream; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java new file mode 100644 index 00000000..32aaf4fc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -0,0 +1,169 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class RuleBuilderService { + + //todo: make this configurable + private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), + new RuleType("MAN"), + new RuleType("X"), + new RuleType("DICT"), + new RuleType("FA"), + new RuleType("LDS"))); + + + public RuleBuilderModel getRuleBuilderModel() { + + RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + + ruleBuilderModel.setWhenClauses(Collections.emptyList()); + ruleBuilderModel.setThenConditions(Collections.emptyList()); + + return ruleBuilderModel; + } + + + public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + log.info("Starting to remove system rules from ruleFile"); + removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); + log.info("Finished removing system rules for ruleFile"); + + return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); + } + + + public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { + + log.info("starting to merge user rules update with system rules"); + RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + mergedRuleFileBlueprint.getRuleClasses() + .forEach(ruleClass -> { + if (systemRules.stream() + .map(RuleType::name) + .toList().contains(ruleClass.ruleType().name())) { + throw new RuntimeException("No system rule updates allowed in user rule update."); + } + }); + removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getRuleClasses() + .stream() + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .flatMap(ruleUnit -> ruleUnit.rules() + .stream())) + .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); + mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() + .stream(), + mergedRuleFileBlueprint.getFunctions() + .stream()) + .toList()); + mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() + .stream(), + mergedRuleFileBlueprint.getDeclarations() + .stream()) + .toList()); + log.info("finished merging user rules update with system rules"); + RuleMergingResult mergingResult = RuleMergingResult.builder() + .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) + .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) + .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .build(); + + return mergingResult; + } + + + private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { + + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + removeAllRulesExceptSystemRules(ruleFileBluePrint); + ruleFileBluePrint.getRuleClasses() + .forEach(ruleClass -> { + if (!systemRuleNames.contains(ruleClass.ruleType().name())) { + throw new RuntimeException("there was an error removing all rules except system rules"); + } + }); + } + + + private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + + List rules = new ArrayList(); + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + + ruleFileBluePrint.getRuleClasses() + .stream() + .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) + .forEach(rule -> rules.add(rule)); + rules.forEach(ruleIdentifier -> { + ruleFileBluePrint.removeRule(ruleIdentifier); + }); + } + + + private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { + + removeSystemRules(ruleFileBluePrint); + for (RuleType systemRule : systemRules) { + List remainingSystemRules = new ArrayList(); + ruleFileBluePrint.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(remainingSystemRules::add)); + if (!remainingSystemRules.isEmpty()) { + throw new RuntimeException("There was an error removing the system rules from the file"); + } + } + } + + + private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { + + for (RuleType systemRule : systemRules) { + List rules = new ArrayList(); + ruleFileBluePrintExisting.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(rules::add)); + rules.forEach(ruleUnit -> { + ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); + }); + ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index 9b983752..e27361be 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -69,51 +69,64 @@ public class DroolsValidationService { private DroolsValidation buildCustomDroolsValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation(); + DroolsValidation customValidation = ruleCompilationResult.getDroolsValidation(); - addSyntaxDeprecatedWarnings(ruleFileBluePrint, customValidation); + addSyntaxDeprecatedWarnings(ruleCompilationResult, customValidation); - addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxErrorMessages(ruleFileType, ruleCompilationResult, customValidation); if (redactionServiceSettings.isRuleExecutionSecured()) { - addBlacklistErrorMessages(ruleFileBluePrint, customValidation); + addBlacklistErrorMessages(ruleCompilationResult, customValidation); } return customValidation; } - private void addSyntaxDeprecatedWarnings(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxDeprecatedWarnings(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { // find deprecated elements in the ruleFileBluePrint - DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); + DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleCompilationResult); if (warningMessageForImports != null) { customValidation.getDeprecatedWarnings().add(warningMessageForImports); } + customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleCompilationResult)); + } + + + private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleCompilationResult ruleCompilationResult) { + + return DroolsSyntaxDeprecatedWarnings.builder() + .message("global ComponentMappingService " + + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL + + "\n is missing from the rules, consider adding it, as it will be required in future versions!") + .line(ruleCompilationResult.getGlobalsLine()) + .column(0) + .build(); customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } - private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { + private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleCompilationResult ruleCompilationResult) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { - String imports = ruleFileBluePrint.getImports(); + String imports = ruleCompilationResult.getImports(); SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation(); List matches = classesSearchImplementation.getMatches(imports); if (!matches.isEmpty()) { String sb = "Following imports are deprecated: \n" + matches.stream() .map(m -> imports.substring(m.startIndex(), m.endIndex())) .collect(Collectors.joining("\n")); - return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build(); + return DroolsSyntaxDeprecatedWarnings.builder().line(ruleCompilationResult.getImportLine()).column(0).message(sb).build(); } } return null; } - private List getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { + private List getWarningsForDeprecatedRules(RuleCompilationResult ruleCompilationResult) { List warningMessages = new ArrayList<>(); @@ -122,7 +135,7 @@ public class DroolsValidationService { SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation(); Map deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap(); - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { List matches = methodsSearchImplementation.getMatches(basicRule.getCode()); @@ -146,32 +159,32 @@ public class DroolsValidationService { } - private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { - RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { - case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); - case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + RuleCompilationResult baseRuleCompilationResult = switch (ruleFileType) { + case ENTITY -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); + case COMPONENT -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseComponentRuleFileString()); }; - if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) { + if (!importsAreValid(baseRuleCompilationResult, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getImportLine()) + .line(ruleCompilationResult.getImportLine()) .column(0) - .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports())) + .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleCompilationResult.getImports())) .build()); } - if (!ruleFileBluePrint.getGlobals().contains(baseRuleFileBluePrint.getGlobals())) { + if (!ruleCompilationResult.getGlobals().contains(baseRuleCompilationResult.getGlobals())) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) - .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals())) + .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleCompilationResult.getGlobals())) .build()); } - baseRuleFileBluePrint.getQueries() + baseRuleCompilationResult.getQueries() .forEach(basicQuery -> { - if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { + if (!validateQueryIsPresent(basicQuery, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(basicQuery.getLine()) @@ -182,7 +195,7 @@ public class DroolsValidationService { }); if (ruleFileType.equals(RuleFileType.ENTITY)) { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; - if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { + if (!validateAgendaGroupIsPresent(ruleCompilationResult, requiredAgendaGroup)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(0) @@ -194,18 +207,18 @@ public class DroolsValidationService { } - private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) { + private boolean validateAgendaGroupIsPresent(RuleCompilationResult ruleCompilationResult, String agendaGroupName) { - return ruleFileBluePrint.streamAllRules() + return ruleCompilationResult.streamAllRules() .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName)); } - private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) { + private boolean importsAreValid(RuleCompilationResult baseRuleCompilationResult, RuleCompilationResult ruleCompilationResult) { // imports may shrink, but not add anything new! - Set baseImports = baseRuleFileBluePrint.getImportSplitByKeyword(); - Set imports = ruleFileBluePrint.getImportSplitByKeyword(); + Set baseImports = baseRuleCompilationResult.getImportSplitByKeyword(); + Set imports = ruleCompilationResult.getImportSplitByKeyword(); Set additionalImports = Sets.difference(imports, baseImports); @@ -214,15 +227,15 @@ public class DroolsValidationService { } - private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) { + private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleCompilationResult ruleCompilationResult) { - return ruleFileBluePrint.getQueries() + return ruleCompilationResult.getQueries() .stream() .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode())); } - private void addBlacklistErrorMessages(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addBlacklistErrorMessages(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { List blacklistErrorMessages = new ArrayList<>(); @@ -234,14 +247,14 @@ public class DroolsValidationService { // check also the imports DroolsBlacklistErrorMessage blacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, - ruleFileBluePrint.getImports(), - ruleFileBluePrint.getImportLine()); + ruleCompilationResult.getImports(), + ruleCompilationResult.getImportLine()); if (blacklistErrorMessage != null) { blacklistErrorMessages.add(blacklistErrorMessage); } // check the rules - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { DroolsBlacklistErrorMessage ruleBlacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java similarity index 53% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index cd6a953e..616cabde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -22,21 +22,48 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.RuleType; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @UtilityClass -public class RuleFileParser { +public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { + + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); + List allRules = new LinkedList<>(); + List allQueries = new LinkedList<>(); + PackageDescr packageDescr = new PackageDescr(); + String imports = ""; + String globals = ""; + List ruleClasses = buildRuleClasses(allRules); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + @SneakyThrows - public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); @@ -52,12 +79,15 @@ public class RuleFileParser { } } - String imports = ruleString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + String imports = ""; + if (!removedImports) { + imports = ruleString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -65,21 +95,81 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + + @SneakyThrows + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { + + return buildRuleCompilationResultFromRuleString(ruleString,false); + } + + + @SneakyThrows + public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { + + StringBuilder ruleStringBuilder = new StringBuilder(); + + // Append imports + ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); + + // Append globals + if (!bluePrint.getGlobals().isEmpty()) { + ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); + } + + // Append queries + for (BasicQuery query : bluePrint.getQueries()) { + ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); + } + + // Append rules + for (RuleClass ruleClass : bluePrint.getRuleClasses()) { + ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); + } + + // Return the final rule string + return ruleStringBuilder.toString().trim(); + } + + + private String buildQueryString(BasicQuery query) { + + return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; + } + + + private String buildRuleString(RuleClass ruleClass) { + + StringBuilder ruleBuilder = new StringBuilder(); + + // Use RuleType to distinguish between different rule types, if needed + ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); + + for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { + for (BasicRule rule : ruleUnit.rules()) { + // Assuming BasicRule has a method to retrieve the condition as a string + ruleBuilder.append(rule.getCode()).append("\n"); + } + } + ruleBuilder.append("end"); + + return ruleBuilder.toString(); } @@ -126,6 +216,7 @@ public class RuleFileParser { private List buildRuleClasses(List allRules) { + //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv diff --git a/redaction-service-v1/rules-management/src/main/resources/order_template.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/order_template.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java new file mode 100644 index 00000000..1996af83 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -0,0 +1,149 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RuleBuilderTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); + private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); + private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + + @Autowired + RuleBuilderService ruleBuilderService; + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import(LayoutParsingServiceProcessorConfiguration.class) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RuleBuilderTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + public void removeSystemRulesTest() { + + String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); + String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + + Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + + Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + } + + + @Test + public void mergeUserRulesUpdateTest() { + + RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); + String mergedRules = mergingResult.getMergedRules(); + RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); + ruleFileBluePrintUserRulesUpdate.getRuleClasses() + .forEach(ruleClass -> { + Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); + }); + + } + + + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getRuleClasses() + .stream() + .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) + .collect(Collectors.toList()).size() == 0; + } + + + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getImports().isEmpty(); + } + + + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getQueries().isEmpty(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java index cac2af47..2d95017d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java @@ -11,10 +11,10 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.core.io.ClassPathResource; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index b8fd95fe..2e42db62 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -22,11 +22,11 @@ import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; import lombok.SneakyThrows; @@ -235,11 +235,11 @@ class DroolsValidationServiceTest { if (droolsValidation.isCompiled()) { continue; } - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(rulesString); + RuleCompilationResult baseRuleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); - rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); - rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + rulesString = rulesString.replace(ruleCompilationResult.getImports(), baseRuleCompilationResult.getImports()); + rulesString = rulesString.replace(ruleCompilationResult.getGlobals(), baseRuleCompilationResult.getGlobals()); try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); @@ -460,9 +460,9 @@ class DroolsValidationServiceTest { end """; - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled()); + assertFalse(ruleCompilationResult.getDroolsValidation().isCompiled()); } } \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java similarity index 91% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java index f8d25b6a..fa77a92b 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,10 +27,10 @@ public class DroolsCompilationTest { @Test public void testValidateRuleSyntax() throws IOException { - URL rmURL = Resources.getResource("all_redact_manager_rules.drl"); + URL rmURL = Resources.getResource("rulesmanagement/all_redact_manager_rules.drl"); String rmRule = Resources.toString(rmURL, StandardCharsets.UTF_8); - URL dmURL = Resources.getResource("all_rules_documine.drl"); + URL dmURL = Resources.getResource("rulesmanagement/all_rules_documine.drl"); String dmRule = Resources.toString(dmURL, StandardCharsets.UTF_8); assertTrue(validateRuleSyntax(rmRule).isValid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java new file mode 100644 index 00000000..577f996a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; + +public class RuleFileBluePrintMergingTest { + + @Test + public void testBothRuleFilesCanBeMerged() { + + RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); + assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); + } + +} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java similarity index 93% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java index 2fc8aa7c..d9657a1c 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import java.io.File; import java.nio.file.Files; @@ -8,7 +8,7 @@ import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.migration.RuleFileMigrator; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java similarity index 94% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java index b6b85cef..ce84ebbf 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -19,10 +19,10 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/addNewRulesHere.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/addNewRulesHere.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl diff --git a/redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Basf-Demo/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Basf-Demo/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_function_tests/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_function_tests/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_rule_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_rule_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_security_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_security_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/PII/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/PII/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/case_insensitive/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/case_insensitive/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/no_redaction_indicator/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/no_redaction_indicator/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/vertebrate_only/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/vertebrate_only/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl new file mode 100644 index 00000000..16ea73d2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl @@ -0,0 +1,50 @@ +package drools +//------------------------------------ Table extraction rules ------------------------------------ + +// Rule unit: TAB.0 +rule "TAB.0.0: Changed Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Changed Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +// Rule unit: TAB.6 +rule "TAB.6.1: Changed Targeted cell extraction (Experimental Stop date)" + when + $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2")) + $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList() + TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList() + $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList() + then + entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date")); + end + diff --git a/redaction-service-v1/rules-management/build.gradle.kts b/redaction-service-v1/rules-management/build.gradle.kts index 2aaea40c..b005ed7e 100644 --- a/redaction-service-v1/rules-management/build.gradle.kts +++ b/redaction-service-v1/rules-management/build.gradle.kts @@ -27,10 +27,10 @@ sourceSets { } dependencies { - implementation(project(":redaction-service-server-v1")) + implementation(project(mapOf("path" to ":redaction-service-server-v1"))) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") - + implementation(project(":redaction-service-server-v1")) implementation("com.github.javaparser:javaparser-core:3.25.3") implementation("org.drools:drools-drl-parser:8.41.0.Final") implementation("org.apache.commons:commons-csv:1.10.0") diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java index 7267bd20..6ab58cf3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java @@ -17,12 +17,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; @@ -72,7 +71,6 @@ public class Main { Set identifiers = switch (type) { case "list", "l" -> RuleIdentifier.fromListOfIdentifiersString(input); case "file", "f" -> RuleFileParser.parseRuleIdentifiersFromFile(input); - case "old", "o" -> OldRulesParser.translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(input), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid", cmd.getOptionValue("t"))); }; ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType); @@ -102,7 +100,6 @@ public class Main { String type = cmd.hasOption("t") ? cmd.getOptionValue("t") : "f"; Map> identifiersPerFile = switch (type) { case "file", "f" -> parseIdentifiersFromFiles(Path.of(cmd.getOptionValue("input"))); - case "old", "o" -> translateIdentifiersFromOldFiles(Path.of(cmd.getOptionValue("input")), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid in combination with the \"recursive\" flag", cmd.getOptionValue("t"))); }; identifiersPerFile.keySet() @@ -137,13 +134,6 @@ public class Main { } - @SneakyThrows - private static Map> translateIdentifiersFromOldFiles(Path inputDirectory, ApplicationType applicationType) { - - return RuleFileIO.streamAllRuleFilesInDirectory(inputDirectory) - .collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, applicationType))); - } - @SneakyThrows private static Map> parseIdentifiersFromFiles(Path inputDirectory) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java deleted file mode 100644 index af844a6d..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -public class RuleManagementResources { - - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_redact_manager_rules.drl"); - } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl"); - } - - - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers.txt"); - } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers_dm.txt"); - } - } - - - public static InputStream getTemplateInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("order_template.txt"); - } - - - public static InputStream getOldRulesCsvInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("old_rules_with_translations.csv"); - } - - - @SneakyThrows - public static String createTempOldRulesCsv(String formattedAsCsv) { - - File csvFile = File.createTempFile("old_rules_with_translations-", ".csv"); - try (var out = new FileOutputStream(csvFile)) { - out.write(formattedAsCsv.getBytes(StandardCharsets.UTF_8)); - } - return csvFile.toString(); - } - -} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java deleted file mode 100644 index ae0f3134..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java +++ /dev/null @@ -1,203 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; -import org.drools.drl.ast.descr.PackageDescr; -import org.drools.drl.ast.descr.RuleDescr; -import org.drools.drl.parser.DrlParser; -import org.kie.internal.builder.conf.LanguageLevelOption; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.OldRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; - -@UtilityClass -public class OldRulesParser { - - static List HEADERS = List.of("id", "old rule names", "old rule code", "translates to"); - - - @SneakyThrows - public void createSetOfTranslatedRuleIdentifiersForEachFileInTheCsv() { - - List allHeaders = List.of(); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - - } - - - public Set translateOldRulesFileToNewIdentifiers(File oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesFileToNewIdentifiers(oldRulesFile.toString(), applicationType); - } - - - public Set translateOldRulesFileToNewIdentifiers(String oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(oldRulesFile), applicationType); - } - - - public Set translateEscapedOldRulesStringToNewIdentifiers(String escapedOldRulesString, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.unescapeAndUnWrap(escapedOldRulesString), applicationType); - } - - - public Set translateOldRulesStringToNewIdentifiers(String oldRulesString, ApplicationType applicationType) { - - List oldRules = parseOldRules(oldRulesString); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - Map> translationPairs = new HashMap<>(); - for (OldRule oldRule : oldRules) { - List translatedIdentifiers = records.stream() - .filter(oldRulesCsvRecord -> oldRulesCsvRecord.code().equals(oldRule.code())) - .map(OldRulesCsvRecord::translatesTo) - .findAny() - .orElse(Collections.emptyList()); - translationPairs.put(oldRule, translatedIdentifiers); - } - boolean allTranslated = true; - for (OldRule oldRule : translationPairs.keySet()) { - if (translationPairs.get(oldRule).isEmpty()) { - allTranslated = false; - records.add(new OldRulesCsvRecord(records.size(), oldRule.code(), oldRule.name(), Collections.emptyList())); - System.out.printf( - "Rule %s has not been translated yet! %nIt has been added to the bottom of the old_rules_with_translations.csv file. To continue, translate the rule and enter its identifier in the csv, then try again!%n", - oldRule.name()); - } - } - if (!allTranslated) { - String formattedAsCsv = formatAsCsv(records); - String fileLocation = RuleManagementResources.createTempOldRulesCsv(formattedAsCsv); - System.out.printf("CSV File with updated values is located at %s%n", fileLocation); - throw new IllegalArgumentException("Non translated Rule found!"); - } - RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); - - translationPairs.forEach((key, value) -> System.out.printf("Rule %s will be translated to %s \n", - key.name(), - String.format("%s: %s", - value, - value.stream() - .map(bluePrint::findRuleByIdentifier) - .flatMap(Collection::stream) - .map(BasicRule::name) - .toList()))); - - return Stream.concat(// - Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),// - translationPairs.values() - .stream() - .flatMap(Collection::stream)) - .map(ruleIdentifier -> new RuleIdentifier(ruleIdentifier.type(), ruleIdentifier.unit(), null)) - .collect(Collectors.toSet()); - } - - - @SneakyThrows - public String formatAsCsv(List records) { - - StringWriter sw = new StringWriter(); - CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS.toArray(String[]::new)).build(); - try (CSVPrinter printer = new CSVPrinter(sw, csvFormat)) { - for (OldRulesCsvRecord record : records) { - printer.printRecord(Stream.of(record.id, RuleFileIO.escapeAndWrap(record.names), RuleFileIO.escapeAndWrap(record.code), record.translatesTo)); - } - } - return sw.toString(); - } - - - @SneakyThrows - private List parseOldRules(String oldRuleString) { - - DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); - PackageDescr packageDescr = parser.parse(false, oldRuleString); - List oldRules = new LinkedList<>(); - for (RuleDescr rule : packageDescr.getRules()) { - if (!rule.isRule()) { - continue; - } - oldRules.add(OldRule.fromRuleDescr(rule, oldRuleString)); - } - return oldRules; - } - - - @SneakyThrows - public List getOldRulesCsvRecords(InputStream allRulesCsvInputStream) { - - List parsedRecords = new LinkedList<>(); - try (Reader reader = new InputStreamReader(allRulesCsvInputStream)) { - Iterable records = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord().parse(reader); - for (CSVRecord record : records) { - String[] values = record.values(); - parsedRecords.add(new OldRulesCsvRecord(Long.parseLong(values[0]), - RuleFileIO.unescapeAndUnWrap(values[2]), - RuleFileIO.unescapeAndUnWrap(values[1]), - parseRuleIdentifiers(values[3]))); - } - } - return parsedRecords; - } - - - private List parseRuleIdentifiers(String value) { - - String cleanedValue = value; - if (cleanedValue.startsWith("[")) { - cleanedValue = cleanedValue.substring(1); - } - if (cleanedValue.endsWith("]")) { - cleanedValue = cleanedValue.substring(0, cleanedValue.length() - 1); - } - if (cleanedValue.isEmpty() || cleanedValue.isBlank()) { - return Collections.emptyList(); - } - List ruleIdentifiers = new LinkedList<>(); - for (String identifier : cleanedValue.split(", ")) { - ruleIdentifiers.add(RuleIdentifier.fromString(identifier)); - } - return ruleIdentifiers; - } - - - private List parseBooleanList(String[] values) { - - return Arrays.stream(values) - .map(Boolean::parseBoolean) - .toList(); - } - - - public record OldRulesCsvRecord(long id, String code, String names, List translatesTo) { - - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java deleted file mode 100644 index 559101ac..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; - -public class RuleFileBluePrintMergingTest { - - @Test - public void testBothRuleFilesCanBeMerged() { - - RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); - assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java deleted file mode 100644 index b78ab534..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -class OldRulesParserTest { - - @Test - @SneakyThrows - @Disabled - public void translateOldRulesToNewIdentifiersTest() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("efsa_regulation_rules.txt")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateEscapedOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @SneakyThrows - @Disabled - public void translateMoreOldRules() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("basf/demo/table_demo.drl")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @Disabled - public void printTranslationsTest() { - - List records = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")) - .forEach(type -> { - List rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type) - .orElseThrow().ruleUnits(); - rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records)); - }); - } - - - private void printOldRulesThatTranslatesToNewRule(RuleUnit ruleUnit, List records) { - - if (ruleUnit.rules().isEmpty()) { - System.out.println("Rule unit empty, skipping!\n"); - return; - } - RuleIdentifier identifier = ruleUnit.rules() - .get(0).identifier(); - RuleIdentifier unitIdentifier = new RuleIdentifier(identifier.type(), identifier.unit(), null); - String oldNames = records.stream() - .filter(r -> r.translatesTo() - .stream() - .anyMatch(i -> i.matches(unitIdentifier))) - .map(OldRulesParser.OldRulesCsvRecord::names) - .map(OldRulesParserTest::removeIdFromName) - .distinct() - .collect(Collectors.joining(", ")); - System.out.println(unitIdentifier.toRuleUnitString() + " " + ruleUnit.rules() - .stream() - .map(BasicRule::name) - .collect(Collectors.joining(", "))); - System.out.println("translate from"); - System.out.println(oldNames); - System.out.println(); - System.out.println(); - } - - - private static String removeIdFromName(String name) { - - String[] values = name.split(":"); - return String.join(":", Arrays.copyOfRange(values, 1, values.length)); - } - - - @Test - @Disabled - @SneakyThrows - public void findAllCustomerRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("pilot/EFSA_sanitisation_GFL_v1").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.RM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.RM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void findAllRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("business-logic/Syngenta_RSS").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.DM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.DM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void translateDossierTemplatesV2() { - -// String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/RED/dossier-templates-v2/"; -// Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), Files.walk(Path.of(dossierTemplatesRepo + "docu")), Files.walk(Path.of(dossierTemplatesRepo + "qa"))) - String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/"; - Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), - Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) - // - ) - .flatMap(Function.identity())// - .filter(path -> path.getFileName().toString().equals("rules.drl"))// - .map(Path::toFile)// - .forEach(this::translateOldRuleFile); - - } - - - @SneakyThrows - private void translateOldRuleFile(File oldRulesFile) { - - Set identifiers = OldRulesParser.translateOldRulesFileToNewIdentifiers(oldRulesFile, ApplicationType.RM); - String newRulesString = RuleFileFactory.createFileFromIdentifiers(identifiers, ApplicationType.RM); - String result = RuleFileIO.escapeAndWrap(newRulesString); - - try (var out = new FileOutputStream(oldRulesFile)) { - out.write(result.getBytes(StandardCharsets.UTF_8)); - } - } - -} \ No newline at end of file -- 2.47.2 From e535c62b6cc48bb0ed3d8043c5586c1988ad146b Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 09:44:51 +0100 Subject: [PATCH 008/108] RED-9472: seperation of system rules code review changes --- .../controller/RuleBuilderController.java | 9 +- .../model/drools/RuleCompilationResult.java | 46 +---- .../RuleManagementResources.java | 19 +- .../factory/RuleFileParser.java | 15 +- .../models/RuleFileBluePrint.java | 28 +++ .../v1/server/service/RuleBuilderService.java | 91 ++------- .../drools/RuleCompilationResultParser.java | 83 +------- .../redaction/v1/server/RuleBuilderTest.java | 183 ++++++++---------- 8 files changed, 147 insertions(+), 327 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 8f3a51dd..aa215fc7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -45,7 +45,7 @@ public class RuleBuilderController implements RuleBuilderResource { RulesResponse rulesResponse = new RulesResponse(); String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); if (filteredRules.isEmpty()) { - throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + throw new AssertionError("There was an error when cleaning the rulefile of sytem rules"); } rulesResponse.setRules(filteredRules); return rulesResponse; @@ -53,13 +53,13 @@ public class RuleBuilderController implements RuleBuilderResource { @Override - public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); DroolsValidationResponse droolsValidationResponse; RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); if (mergingResult.getMergedRules().isEmpty()) { - throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); @@ -90,7 +90,8 @@ public class RuleBuilderController implements RuleBuilderResource { .toList()) .build(); if (!droolsValidation.isCompiled()) { - return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.UNPROCESSABLE_ENTITY); } else { // rulesUploadResponse.setRules(mergingResult.getMergedRules()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index e96aa697..3506a31f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -49,60 +49,16 @@ public final class RuleCompilationResult { public void dropRulesByIdentifier(RuleType ruleType) { - log.info("removing rule with identifier {}", ruleType.name()); - List rulesToBeRemoved = ruleClasses.stream() .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); ruleClasses.removeAll(rulesToBeRemoved); } - public void dropAllRulesExceptSystemRules(List systemRules) { - - log.info("removing all rules except the system rules"); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) - .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); - ruleClasses.removeAll(rulesToBeRemoved); - } - - - public void dropImports() { - - this.imports = ""; - } - - - public void dropQueries() { - - this.queries.clear(); - } - - - public int countRuleOccurences(RuleType ruleType) { - - log.info("counting occurences of files {}", ruleType.name()); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) - .toList(); - return rulesToBeRemoved.size(); - } - - - private boolean filterOutRule(RuleType ruleType, List filteredRules) { - - return !filteredRules.contains(ruleType.name()); - } - - public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replace("\n", "").split("import")) + return Arrays.stream(imports.replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java index 331b1e25..e423dde8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -1,34 +1,37 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement; +import java.io.IOException; import java.io.InputStream; +import org.drools.io.ClassPathResource; + import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; @SuppressWarnings("PMD") public class RuleManagementResources { - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + public static InputStream getAllRulesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + return new ClassPathResource("drools/all_redact_manager_rules.drl").getInputStream(); } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + return new ClassPathResource("drools/all_rules_documine.drl").getInputStream(); } - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers.txt").getInputStream(); } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers_dm.txt").getInputStream(); } } - public static InputStream getTemplateInputStream() { + public static InputStream getTemplateInputStream() throws IOException { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + return new ClassPathResource("rulesmanagement/order_template.txt").getInputStream(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index db0ece77..6354f93c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -4,7 +4,6 @@ import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -121,16 +120,12 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - if (rulesPerUnit.keySet() != null) { - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); - } else { - return new ArrayList<>(); - } + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 28b2663c..ca185b63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; @@ -53,6 +54,33 @@ public class RuleFileBluePrint { } + public boolean removeRuleClassByRuleIdentifier(RuleIdentifier ruleIdentifier) { + + AtomicBoolean wasRemoved = new AtomicBoolean(false); + + findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> { + List unitsToRemove = new ArrayList<>(); + + ruleClass.ruleUnits() + .forEach(ruleUnit -> { + boolean removed = ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier)); + if (removed) { + wasRemoved.set(true); + if (ruleUnit.rules().isEmpty()) { + unitsToRemove.add(ruleUnit); + } + } + }); + + ruleClass.ruleUnits().removeAll(unitsToRemove); + if (ruleClass.ruleUnits().isEmpty()) { + this.ruleClasses.remove(ruleClass); + } + }); + return wasRemoved.get(); + } + + public Optional findRuleClassByType(RuleType ruleType) { return ruleClasses.stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 32aaf4fc..d20e8337 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -3,9 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -16,8 +15,6 @@ import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFil import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.extern.slf4j.Slf4j; @@ -25,13 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - //todo: make this configurable - private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), - new RuleType("MAN"), - new RuleType("X"), - new RuleType("DICT"), - new RuleType("FA"), - new RuleType("LDS"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); public RuleBuilderModel getRuleBuilderModel() { @@ -48,28 +44,24 @@ public class RuleBuilderService { public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - log.info("Starting to remove system rules from ruleFile"); - removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); - log.info("Finished removing system rules for ruleFile"); - + removeSystemRules(ruleFileBluePrint); return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); } public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { - log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() - .map(RuleType::name) - .toList().contains(ruleClass.ruleType().name())) { - throw new RuntimeException("No system rule updates allowed in user rule update."); + .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { + log.warn("System rule in user rule update."); + //throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); ruleFileBluePrintExisting.getRuleClasses() .stream() .flatMap(ruleClass -> ruleClass.ruleUnits() @@ -89,7 +81,6 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - log.info("finished merging user rules update with system rules"); RuleMergingResult mergingResult = RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) @@ -100,69 +91,17 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { - - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); - removeAllRulesExceptSystemRules(ruleFileBluePrint); - ruleFileBluePrint.getRuleClasses() - .forEach(ruleClass -> { - if (!systemRuleNames.contains(ruleClass.ruleType().name())) { - throw new RuntimeException("there was an error removing all rules except system rules"); - } - }); - } - - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - List rules = new ArrayList(); - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); - ruleFileBluePrint.getRuleClasses() - .stream() - .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) - .forEach(rule -> rules.add(rule)); - rules.forEach(ruleIdentifier -> { - ruleFileBluePrint.removeRule(ruleIdentifier); - }); - } - - - private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { - - removeSystemRules(ruleFileBluePrint); - for (RuleType systemRule : systemRules) { - List remainingSystemRules = new ArrayList(); - ruleFileBluePrint.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(remainingSystemRules::add)); - if (!remainingSystemRules.isEmpty()) { - throw new RuntimeException("There was an error removing the system rules from the file"); - } - } } private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { - for (RuleType systemRule : systemRules) { - List rules = new ArrayList(); - ruleFileBluePrintExisting.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(rules::add)); - rules.forEach(ruleUnit -> { - ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); - }); - ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + for (RuleIdentifier systemRule : systemRules) { + ruleFileBluePrintExisting.removeRuleClassByRuleIdentifier(systemRule); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 616cabde..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -35,33 +35,6 @@ public class RuleCompilationResultParser { "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { - - DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); - List allRules = new LinkedList<>(); - List allQueries = new LinkedList<>(); - PackageDescr packageDescr = new PackageDescr(); - String imports = ""; - String globals = ""; - List ruleClasses = buildRuleClasses(allRules); - return new RuleCompilationResult(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); - } - - @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { @@ -116,60 +89,7 @@ public class RuleCompilationResultParser { @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { - return buildRuleCompilationResultFromRuleString(ruleString,false); - } - - - @SneakyThrows - public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { - - StringBuilder ruleStringBuilder = new StringBuilder(); - - // Append imports - ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); - - // Append globals - if (!bluePrint.getGlobals().isEmpty()) { - ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); - } - - // Append queries - for (BasicQuery query : bluePrint.getQueries()) { - ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); - } - - // Append rules - for (RuleClass ruleClass : bluePrint.getRuleClasses()) { - ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); - } - - // Return the final rule string - return ruleStringBuilder.toString().trim(); - } - - - private String buildQueryString(BasicQuery query) { - - return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; - } - - - private String buildRuleString(RuleClass ruleClass) { - - StringBuilder ruleBuilder = new StringBuilder(); - - // Use RuleType to distinguish between different rule types, if needed - ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); - - for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { - for (BasicRule rule : ruleUnit.rules()) { - // Assuming BasicRule has a method to retrieve the condition as a string - ruleBuilder.append(rule.getCode()).append("\n"); - } - } - ruleBuilder.append("end"); - - return ruleBuilder.toString(); + return buildRuleCompilationResultFromRuleString(ruleString, false); } @@ -216,7 +136,6 @@ public class RuleCompilationResultParser { private List buildRuleClasses(List allRules) { - //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 1996af83..181e108a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,149 +1,128 @@ package com.iqser.red.service.redaction.v1.server; +import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; import static org.mockito.Mockito.when; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; -import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; -import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class RuleBuilderTest extends AbstractRedactionIntegrationTest { +import org.junit.jupiter.api.Assertions; +import lombok.SneakyThrows; - private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); - private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); - private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); - - @Autowired - RuleBuilderService ruleBuilderService; - - @Configuration - @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) - @Import(LayoutParsingServiceProcessorConfiguration.class) - @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) - public static class RuleBuilderTestConfiguration { - - @Bean - @Primary - public StorageService inmemoryStorage() { - - return new FileSystemBackedStorageService(ObjectMapperFactory.create()); - } - - } +class RuleBuilderTest { + private RuleBuilderService ruleBuilderService; + private final List systemRules = new ArrayList<>(Arrays.asList( + RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*") + )); @BeforeEach - public void stubClients() { - + void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); - - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); - - loadDictionaryForTest(); - loadTypeForTest(); - loadNerForTest(); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); - - when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); - - mockDictionaryCalls(null); - - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } - @Test - public void removeSystemRulesTest() { + @SneakyThrows + void removeSystemRulesTest() throws IOException { + URL url = getClass().getClassLoader().getResource("drools"); + if (url == null) { + throw new RuntimeException("Drools directory not found in resources"); + } - String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); - String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); - RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + Path path = Paths.get(url.toURI()); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); - Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); - - Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } + private void testThatSystemRulesAreRemoved(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + true + ); - @Test - public void mergeUserRulesUpdateTest() { + RuleCompilationResult ruleCompilationResultWithImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); - RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); - String mergedRules = mergingResult.getMergedRules(); - RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); - ruleFileBluePrintUserRulesUpdate.getRuleClasses() - .forEach(ruleClass -> { - Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); - }); + Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports + ); + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + + Assertions.assertEquals( + RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + } private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getRuleClasses() - .stream() - .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) - .collect(Collectors.toList()).size() == 0; + .stream() + .map(RuleClass::ruleType) + .noneMatch(ruleType -> systemRules.stream() + .map(RuleIdentifier::type) + .anyMatch(type -> ruleType.name().equals(type.name()))); } - private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getImports().isEmpty(); } - private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getQueries().isEmpty(); } - -} +} \ No newline at end of file -- 2.47.2 From cb102a4e00c3444da1c67e430ef5a6a490f1231e Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 009/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 7fcaaacded22ce75d1a4508e33089204ed01f2d8 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 010/108] RED-9472: seperation of system rules pmd fix --- .../v1/server/service/RuleBuilderService.java | 20 +++++++++---------- .../drools/RuleCompilationResultParser.java | 8 ++------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index d20e8337..2c9c9f7e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -58,7 +58,7 @@ public class RuleBuilderService { if (systemRules.stream() .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { log.warn("System rule in user rule update."); - //throw new RuntimeException("No system rule updates allowed in user rule update."); + throw new RuntimeException("No system rule updates allowed in user rule update."); } }); removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); @@ -81,19 +81,17 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - RuleMergingResult mergingResult = RuleMergingResult.builder() + return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) .build(); - - return mergingResult; } private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 0342c20aabf6f8727845b47b497d4e5dc8847d19 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:26:32 +0100 Subject: [PATCH 011/108] RED-9472: seperation of system rules pmd fix --- .../redaction/v1/server/RuleBuilderTest.java | 83 ++++++++----------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 181e108a..31d61e1e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server; -import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URL; import java.nio.file.Path; @@ -13,8 +8,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; @@ -27,35 +24,32 @@ import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.knecon.fforesight.tenantcommons.TenantContext; -import org.junit.jupiter.api.Assertions; import lombok.SneakyThrows; class RuleBuilderTest { private RuleBuilderService ruleBuilderService; - private final List systemRules = new ArrayList<>(Arrays.asList( - RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*") - )); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); + @BeforeEach void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); } + @Test @SneakyThrows void removeSystemRulesTest() throws IOException { - URL url = getClass().getClassLoader().getResource("drools"); - if (url == null) { - throw new RuntimeException("Drools directory not found in resources"); - } + URL url = Thread.currentThread().getContextClassLoader().getResource("drools"); Path path = Paths.get(url.toURI()); RuleFileIO.streamAllRuleFilesInDirectory(path) .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); @@ -64,20 +58,14 @@ class RuleBuilderTest { .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } - private void testThatSystemRulesAreRemoved(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - true - ); - RuleCompilationResult ruleCompilationResultWithImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + private void testThatSystemRulesAreRemoved(Path path) { + + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), true); + + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); @@ -88,28 +76,22 @@ class RuleBuilderTest { Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - cleanedRulesWithImports - ); + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports); - Assertions.assertEquals( - RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(cleanedRuleFileMergedWithSystemRules.getMergedRules()); + + Assertions.assertEquals(RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), cleanedRuleFileMergedWithSystemRules.getMergedRules()); } + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getRuleClasses() .stream() .map(RuleClass::ruleType) @@ -118,11 +100,16 @@ class RuleBuilderTest { .anyMatch(type -> ruleType.name().equals(type.name()))); } + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getImports().isEmpty(); } + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getQueries().isEmpty(); } + } \ No newline at end of file -- 2.47.2 From 65cee5d425e1be55590a6e06af35baf407a862f7 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 15 Nov 2024 09:45:56 +0100 Subject: [PATCH 012/108] RED-9472: seperation of system rules review changes --- .../models/RuleFileBluePrint.java | 2 +- .../v1/server/service/RuleBuilderService.java | 46 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index ca185b63..533492e1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -180,7 +180,7 @@ public class RuleFileBluePrint { public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), declarations, functions); ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 2c9c9f7e..c346429c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -1,10 +1,10 @@ package com.iqser.red.service.redaction.v1.server.service; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), - RuleIdentifier.fromName("MAN"), - RuleIdentifier.fromName("X"), - RuleIdentifier.fromName("DICT"), - RuleIdentifier.fromName("FA"), - RuleIdentifier.fromName("LDS"))); + private final HashSet systemRules = new HashSet<>(Set.of(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -52,7 +52,7 @@ public class RuleBuilderService { public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() @@ -61,25 +61,30 @@ public class RuleBuilderService { throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - ruleFileBluePrintExisting.getRuleClasses() - .stream() - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .flatMap(ruleUnit -> ruleUnit.rules() - .stream())) + ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getAllRules() .forEach(mergedRuleFileBlueprint::addRule); - mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); - mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setImports(uniqueImports.stream() + .collect(Collectors.joining(""))); + Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setGlobals(uniqueGlobals.stream() + .collect(Collectors.joining(""))); mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() .stream(), mergedRuleFileBlueprint.getFunctions() .stream()) + .distinct() .toList()); mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() .stream(), mergedRuleFileBlueprint.getDeclarations() .stream()) + .distinct() .toList()); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) @@ -89,10 +94,9 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + private RuleFileBluePrint removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + return ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } -- 2.47.2 From 4e7738503338f5d6bf719af68248f20840e3ee17 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 09:04:46 +0100 Subject: [PATCH 013/108] RED-9472: seperation of system rules added document dependency --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 277f9fde..cbefd926 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -38,7 +38,7 @@ configurations.all { } dependencies { - + implementation(project(":document")) implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") -- 2.47.2 From 8c10e1c00d9430910cf95db590b2c725182832e9 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 13:02:03 +0100 Subject: [PATCH 014/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index cbefd926..594dc06a 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.585.0-RED9472.1" +val persistenceServiceVersion = "2.627.0-RED9472.2" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" @@ -45,6 +45,7 @@ dependencies { { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } + implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}") implementation("com.iqser.red.commons:spring-commons:6.2.0") @@ -62,6 +63,7 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") implementation("org.ahocorasick:ahocorasick:0.6.3") + implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") -- 2.47.2 From 447210151ffd98947f2505670345a80878aa77e5 Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 08:52:32 +0100 Subject: [PATCH 015/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 594dc06a..45858e36 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.627.0-RED9472.2" +val persistenceServiceVersion = "2.628.0-RED9472.0" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" -- 2.47.2 From b326000c80e628ee04ed27d20eb6f7d17f69ecaa Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Sep 2024 10:24:59 +0200 Subject: [PATCH 016/108] RED-9472: seperation of system rules added endpoints for seperating and readding system rules working on service RED-9472: seperation of system rules refactored endpoints. now removing system rules on download merging user rules with systemrules on upload RED-9472: seperation of system rules RED-9472: seperation of system rules RED-9472: seperation of system rules refactored two have a rulefileblueprint and a rulefileblueprint for validationresult changed the flow of merging and seperating system and user rules added functions and declarations to rule file parsing RED-9472: seperation of system rules RED-9472: seperation of system rules introduced tests working on fixing bug with merging rules RED-9472: seperation of system rules introduced tests RED-9472: seperation of system rules fixed circular depdency RED-9472: seperation of system rules updated persistence service version RED-9472: seperation of system rules refactored rules refactored pcackages RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed some sonarqube errors RED-9472: seperation of system rules fixed pmd error RED-9472: seperation of system rules added deploy stage to gitlab pipeline RED-9472: seperation of system rules RED-9472: seperation of system rules fixed a mistake RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse --- .../redaction-service-api-v1/build.gradle.kts | 3 +- .../v1/resources/RuleBuilderResource.java | 17 +- .../build.gradle.kts | 32 ++- .../controller/RuleBuilderController.java | 96 ++++++++- .../v1/server/model/RuleMergingResult.java | 24 +++ ...ePrint.java => RuleCompilationResult.java} | 67 +++++- .../RuleManagementResources.java | 34 +++ .../DefaultRuleIdentifiersFactory.java | 8 +- .../factory/RuleFileFactory.java | 51 +++-- .../factory/RuleFileParser.java | 82 +++++-- .../migration/RuleFileMigrator.java | 15 +- .../migration/RuleIdentifierMigrator.java | 45 ++-- .../models/ApplicationType.java | 2 +- .../models/BasicDeclaration.java | 34 +++ .../rulesmanagement/models/BasicFunction.java | 36 ++++ .../rulesmanagement}/models/BasicRule.java | 4 +- .../rulesmanagement}/models/OldRule.java | 2 +- .../rulesmanagement}/models/RuleClass.java | 2 +- .../models/RuleFileBluePrint.java | 42 +++- .../models/RuleIdentifier.java | 2 +- .../rulesmanagement}/models/RuleType.java | 2 +- .../rulesmanagement}/models/RuleUnit.java | 2 +- .../rulesmanagement}/utils/RuleFileIO.java | 2 +- .../v1/server/service/RuleBuilderService.java | 169 +++++++++++++++ .../drools/DroolsValidationService.java | 85 ++++---- ....java => RuleCompilationResultParser.java} | 139 +++++++++--- .../all_redact_manager_rules.drl | 0 .../rulesmanagement}/all_rules_documine.drl | 0 .../default_rule_identifiers.txt | 0 .../default_rule_identifiers_dm.txt | 0 .../old_rules_with_translations.csv | 0 .../rulesmanagement}/order_template.txt | 0 .../redaction/v1/server/RuleBuilderTest.java | 149 +++++++++++++ .../services/DroolsUpToDateTest.java | 8 +- .../services/DroolsValidationServiceTest.java | 16 +- .../DroolsCompilationTest.java | 6 +- .../RuleFileBluePrintMergingTest.java | 20 ++ .../RuleFileMigrationTest.java | 4 +- .../factory/RuleFileFactoryTest.java | 10 +- .../EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rulesmanagement}/addNewRulesHere.drl | 0 .../rulesmanagement}/demo/table_demo.drl | 0 .../rulesmanagement}/dev/Basf-Demo/rules.txt | 0 .../rulesmanagement}/dev/Flora/rules.txt | 0 .../dev/EFSA_Regulation_2021/rules.txt | 0 .../dev/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rules.txt | 0 .../EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../dev/Manual_Redaction/rules.txt | 0 .../docu/EFSA_Regulation_2021/rules.txt | 0 .../docu/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../docu/Manual_Redaction/rules.txt | 0 .../qa/EFSA_Regulation_2021/rules.txt | 0 .../qa/EFSA_Regulation_2022/rules.txt | 0 .../qa/EFSA_acceptance_test/rules.txt | 0 .../qa/EFSA_function_tests/rules.txt | 0 .../qa/EFSA_rule_test/rules.txt | 0 .../qa/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rules.txt | 0 .../qa/EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../qa/EFSA_security_test/rules.txt | 0 .../qa/EFSA_smoke_tests/rules.txt | 0 .../qa/Manual_Redaction/rules.txt | 0 .../dossier_templates_v2/qa/PII/rules.txt | 0 .../qa/case_insensitive/rules.txt | 0 .../qa/no_redaction_indicator/rules.txt | 0 .../qa/vertebrate_only/rules.txt | 0 .../efsa_regulation_rules.txt | 0 .../userrules/user_rule_update.drl | 50 +++++ .../rules-management/build.gradle.kts | 4 +- .../utility/rules/management/Main.java | 20 +- .../management/RuleManagementResources.java | 56 ----- .../translation/OldRulesParser.java | 203 ------------------ .../RuleFileBluePrintMergingTest.java | 20 -- .../translation/OldRulesParserTest.java | 176 --------------- 76 files changed, 1065 insertions(+), 674 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/{RuleFileBluePrint.java => RuleCompilationResult.java} (61%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/DefaultRuleIdentifiersFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileParser.java (50%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleFileMigrator.java (77%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleIdentifierMigrator.java (76%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/ApplicationType.java (73%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/BasicRule.java (80%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/OldRule.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleClass.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleFileBluePrint.java (85%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleIdentifier.java (97%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleType.java (95%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleUnit.java (54%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/utils/RuleFileIO.java (96%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/{RuleFileParser.java => RuleCompilationResultParser.java} (53%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_redact_manager_rules.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_rules_documine.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers_dm.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/old_rules_with_translations.csv (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/order_template.txt (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/DroolsCompilationTest.java (91%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/RuleFileMigrationTest.java (93%) rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactoryTest.java (94%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/addNewRulesHere.drl (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/demo/table_demo.drl (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dev/Basf-Demo/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dev/Flora/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/dev/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/docu/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_function_tests/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_rule_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_security_test/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/Manual_Redaction/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/PII/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/case_insensitive/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/no_redaction_indicator/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/dossier_templates_v2/qa/vertebrate_only/rules.txt (100%) rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/efsa_regulation_rules.txt (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index a4790e05..ccff795e 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,11 +4,12 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.612.0-RED10072.1" +val persistenceServiceVersion = "2.585.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + api("com.knecon.fforesight:swagger-commons:0.7.0") } publishing { diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index 3864ba9b..4683fd5f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,13 +1,26 @@ package com.iqser.red.service.redaction.v1.resources; -import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; - import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; public interface RuleBuilderResource { @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE) RuleBuilderModel getRuleBuilderModel(); + + @PostMapping(value = "/internal-api/rules/user-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithoutSystemRules(@RequestBody SystemRulesSeperationRequest systemRulesSeperationRequest); + + + @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + ResponseEntity mergeUserUpdateRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); + } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 2d67fb11..277f9fde 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -12,12 +12,12 @@ plugins { description = "redaction-service-server-v1" -val layoutParserVersion = "0.191.0" +val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.612.0-RED10072.1" -val llmServiceVersion = "1.20.0-RED10072.2" +val persistenceServiceVersion = "2.585.0-RED9472.1" +val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" @@ -31,10 +31,15 @@ configurations { } } +configurations.all { + resolutionStrategy { + force("com.google.protobuf:protobuf-java:4.27.1") + } +} + dependencies { implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } - implementation(project(":document")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { @@ -47,7 +52,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.50.0") - implementation("com.knecon.fforesight:tenant-commons:0.31.0") + implementation("com.knecon.fforesight:tenant-commons:0.30.0") implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } @@ -56,14 +61,13 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") - implementation("org.ahocorasick:ahocorasick:0.9.0") - implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") - implementation("com.github.roklenarcic:aho-corasick:1.2") + implementation("org.ahocorasick:ahocorasick:0.6.3") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") implementation("org.drools:drools-mvel:${droolsVersion}") implementation("org.kie:kie-spring:7.74.1.Final") + implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.locationtech.jts:jts-core:1.19.0") @@ -79,6 +83,7 @@ dependencies { implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") + api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") @@ -125,7 +130,6 @@ tasks.named("bootBuildImage") { "BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5" ) - environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE! if (project.hasProperty("buildbootDockerHostNetwork")) { @@ -180,19 +184,13 @@ tasks.register("generateJavaDoc", Javadoc::class) { dependsOn("compileJava") dependsOn("delombok") classpath = project.sourceSets["main"].runtimeClasspath - val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { + source = fileTree("${buildDir}/generated/sources/delombok/java/main") { include(droolsImports) } - val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { - include(droolsImports) - } - source = documentFiles + mainFiles - - setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "")) + destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "") options.memberLevel = JavadocMemberLevel.PUBLIC (options as StandardJavadocDocletOptions).apply { title = "API Documentation for Redaction Service ${project.version}" } } - diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 306dccbe..8f3a51dd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,27 +1,111 @@ package com.iqser.red.service.redaction.v1.server.controller; -import java.util.Collections; - +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleBlacklistErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxWarningMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @RestController +@Slf4j @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { + private final RuleBuilderService ruleBuilderService; + private final DroolsValidationService droolsValidationService; + + @Override public RuleBuilderModel getRuleBuilderModel() { - RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + return this.ruleBuilderService.getRuleBuilderModel(); + } - ruleBuilderModel.setWhenClauses(Collections.emptyList()); - ruleBuilderModel.setThenConditions(Collections.emptyList()); - return ruleBuilderModel; + @Override + public RulesResponse getRuleFileWithoutSystemRules(SystemRulesSeperationRequest systemRulesSeperationRequest) { + + RulesResponse rulesResponse = new RulesResponse(); + String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); + if (filteredRules.isEmpty()) { + throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + } + rulesResponse.setRules(filteredRules); + return rulesResponse; + } + + + @Override + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + + RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); + DroolsValidationResponse droolsValidationResponse; + RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + if (mergingResult.getMergedRules().isEmpty()) { + throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + } + try { + var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + droolsValidationResponse = DroolsValidationResponse.builder() + .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() + .stream() + .map(droolsSyntaxErrorMessage -> new RuleSyntaxErrorMessage(droolsSyntaxErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxErrorMessage.getColumn(), + droolsSyntaxErrorMessage.getMessage())) + .toList()) + .deprecatedWarnings(droolsValidation.getDeprecatedWarnings() + .stream() + .map(droolsSyntaxDeprecatedWarnings -> new RuleSyntaxWarningMessage(droolsSyntaxDeprecatedWarnings.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxDeprecatedWarnings.getColumn(), + droolsSyntaxDeprecatedWarnings.getMessage())) + .toList()) + .blacklistErrorMessages(droolsValidation.getBlacklistErrorMessages() + .stream() + .map(droolsBlacklistErrorMessage -> new RuleBlacklistErrorMessage(droolsBlacklistErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsBlacklistErrorMessage.getColumn(), + droolsBlacklistErrorMessage.getMessage())) + .toList()) + .build(); + if (!droolsValidation.isCompiled()) { + return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + } else { + // + rulesUploadResponse.setRules(mergingResult.getMergedRules()); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + } + } catch (Exception e) { + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + } + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); + } + + + private enum RuleFileType { + ENTITY, + COMPONENT } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java new file mode 100644 index 00000000..b2cb1b27 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Schema(description = "Object containing a string of Drools rules.") +public class RuleMergingResult { + + @Schema(description = "The merged rules.") + private String mergedRules; + + @Schema(description = "the length of added imports from sytemRules") + private int addedImportsOffset; + @Schema(description = "the length of added globals from sytemRules") + private int addedGlobalsOffset; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java similarity index 61% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index 3aa1cdef..e96aa697 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -20,13 +20,15 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Builder @NoArgsConstructor @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public final class RuleFileBluePrint { +public final class RuleCompilationResult { String imports; int importLine; @@ -45,9 +47,62 @@ public final class RuleFileBluePrint { } + public void dropRulesByIdentifier(RuleType ruleType) { + + log.info("removing rule with identifier {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropAllRulesExceptSystemRules(List systemRules) { + + log.info("removing all rules except the system rules"); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropImports() { + + this.imports = ""; + } + + + public void dropQueries() { + + this.queries.clear(); + } + + + public int countRuleOccurences(RuleType ruleType) { + + log.info("counting occurences of files {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + return rulesToBeRemoved.size(); + } + + + private boolean filterOutRule(RuleType ruleType, List filteredRules) { + + return !filteredRules.contains(ruleType.name()); + } + + public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replaceAll("\n", "").split("import")) + return Arrays.stream(imports.replace("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } @@ -76,7 +131,7 @@ public final class RuleFileBluePrint { public List getAllRuleIdentifiers() { return streamAllRules().map(BasicRule::getIdentifier) - .collect(Collectors.toList()); + .toList(); } @@ -96,4 +151,10 @@ public final class RuleFileBluePrint { return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']'; } + + public void addRuleClass(RuleClass ruleClass) { + + this.ruleClasses.add(ruleClass); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java new file mode 100644 index 00000000..331b1e25 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import java.io.InputStream; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; + +@SuppressWarnings("PMD") +public class RuleManagementResources { + + public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + } + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + } + + + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + } else { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + } + } + + + public static InputStream getTemplateInputStream() { + + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java index 2a8de641..c83f7f04 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java @@ -1,12 +1,12 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java index 039664bf..f6332332 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.Collections.emptySet; @@ -9,13 +9,13 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -48,17 +48,17 @@ public class RuleFileFactory { } RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers); - return buildRuleString(filteredBluePrint); + return buildRuleString(filteredBluePrint, false, false); } @SneakyThrows - public String buildRuleString(RuleFileBluePrint bluePrint) { + public String buildRuleString(RuleFileBluePrint bluePrint, boolean dropImports, boolean dropQueries) { try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) { String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8); List templateRuleOrder = parseRuleOrder(template); - return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder); + return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder, dropImports, dropQueries); } } @@ -72,23 +72,35 @@ public class RuleFileFactory { } - private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder) { + private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder, boolean dropImports, boolean dropQueries) { - Set additionalRuleTypes = bluePrint.ruleClasses() + Set additionalRuleTypes = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleType) .filter(ruleType -> !ruleOrder.contains(ruleType)) .collect(Collectors.toSet()); StringBuilder sb = new StringBuilder(); - sb.append(bluePrint.imports()); + if (!dropImports) { + sb.append(bluePrint.getImports()); + sb.append("\n\n"); + } else { + //todo: this is hacked to enable compiling the rules without imports + sb.append("package drools"); + sb.append("\n\n"); + } + sb.append(bluePrint.getGlobals()); sb.append("\n\n"); - sb.append(bluePrint.globals()); + sb.append("//------------------------------------ declarations ------------------------------------"); sb.append("\n\n"); - sb.append("//------------------------------------ queries ------------------------------------"); - sb.append("\n\n"); - sb.append(bluePrint.queries()); + sb.append(bluePrint.getDeclarations().isEmpty() ? "" : bluePrint.getDeclarations()); sb.append("\n\n"); + if (!dropQueries) { + sb.append("//------------------------------------ queries ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getQueries()); + sb.append("\n\n"); + } for (RuleType ruleBlockType : ruleOrder) { if (ruleBlockType.isWildCard()) { additionalRuleTypes.stream() @@ -101,6 +113,9 @@ public class RuleFileFactory { } writeRuleClass(bluePrint, ruleBlockType, sb); } + sb.append("//------------------------------------ functions ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getFunctions().isEmpty() ? "" : bluePrint.getFunctions()); return sb.toString().trim() + "\n"; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java similarity index 50% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index 3e17b0db..db0ece77 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -1,30 +1,35 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; +import org.drools.drl.ast.descr.FunctionDescr; import org.drools.drl.ast.descr.ImportDescr; import org.drools.drl.ast.descr.PackageDescr; import org.drools.drl.ast.descr.RuleDescr; import org.drools.drl.parser.DrlParser; import org.kie.internal.builder.conf.LanguageLevelOption; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicDeclaration; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicFunction; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -45,10 +50,19 @@ public class RuleFileParser { @SneakyThrows public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString) { + return buildBluePrintFromRulesString(rulesString, false); + } + + + @SneakyThrows + public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString, boolean removeImports) { + DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); PackageDescr packageDescr = parser.parse(false, rulesString); StringBuilder queryBuilder = new StringBuilder(); List allRules = new LinkedList<>(); + List functions = new LinkedList<>(); + List declarations = new LinkedList<>(); for (RuleDescr rule : packageDescr.getRules()) { if (rule.isQuery()) { queryBuilder.append(rulesString, rule.getStartCharacter(), rule.getEndCharacter()); @@ -57,12 +71,30 @@ public class RuleFileParser { } allRules.add(BasicRule.fromRuleDescr(rule, rulesString)); } - String imports = rulesString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + for (FunctionDescr function : packageDescr.getFunctions()) { + functions.add(BasicFunction.fromFunctionDescr(function, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getEnumDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getClassAndEnumDeclarationDescrs()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + String imports = ""; + if (!removeImports && !packageDescr.getImports().isEmpty()) { + imports = rulesString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -70,7 +102,7 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses); + return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses, declarations, functions); } @@ -89,11 +121,17 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); + if (rulesPerUnit.keySet() != null) { + + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); + } else { + return new ArrayList<>(); + } + } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java similarity index 77% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java index c002fed3..afbdff26 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java @@ -1,15 +1,16 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.File; import java.io.FileOutputStream; import java.nio.charset.StandardCharsets; import java.util.List; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -30,7 +31,7 @@ public class RuleFileMigrator { //replaceRules(ruleFileBluePrint, combinedBluePrint); replaceRuleIdentifiers(combinedBluePrint, ruleFileBluePrint); - String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); + String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java similarity index 76% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java index fc8e9026..3889fad1 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.FileOutputStream; import java.io.IOException; @@ -9,16 +9,15 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -35,7 +34,6 @@ public class RuleIdentifierMigrator { @SneakyThrows public void migrateAllRuleIdentifiers() { - List parsedRecords = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); RuleFileBluePrint bluePrint = getBluePrint(ApplicationType.RM); // migrateIdentifier(RuleIdentifier.fromString("PII.10.0"), RuleIdentifier.fromString("CBI.20.0"), bluePrint, parsedRecords); @@ -46,20 +44,17 @@ public class RuleIdentifierMigrator { bluePrint = migrateMatchedRuleForAllRules(bluePrint); - String ruleString = RuleFileFactory.buildRuleString(bluePrint); + String ruleString = RuleFileFactory.buildRuleString(bluePrint, false, false); try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) { out.write(ruleString.getBytes(StandardCharsets.UTF_8)); } - String csvString = OldRulesParser.formatAsCsv(parsedRecords); - try (var out = new FileOutputStream("/tmp/old_rules_with_translations.csv")) { - out.write(csvString.getBytes(StandardCharsets.UTF_8)); - } } + //todo: introduce functions and declarations private static RuleFileBluePrint migrateMatchedRuleForAllRules(RuleFileBluePrint bluePrint) { - List migratedRules = bluePrint.ruleClasses() + List migratedRules = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) @@ -67,7 +62,12 @@ public class RuleIdentifierMigrator { .flatMap(Collection::stream) .map(RuleIdentifierMigrator::migrateMatchedRule) .toList(); - RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.imports(), bluePrint.globals(), bluePrint.queries(), new LinkedList<>()); + RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.getImports(), + bluePrint.getGlobals(), + bluePrint.getQueries(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>()); migratedRules.forEach(migratedBluePrint::addRule); return migratedBluePrint; } @@ -118,7 +118,7 @@ public class RuleIdentifierMigrator { } - public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List records) { + public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint) { BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()) .orElseThrow().findRuleUnitByInteger(oldIdentifier.unit()) @@ -132,9 +132,6 @@ public class RuleIdentifierMigrator { bluePrint.removeRule(oldIdentifier); bluePrint.addRule(newRule); - records.stream() - .filter(record -> record.translatesTo().contains(oldIdentifier)) - .forEach(record -> replaceOldIdentifier(oldIdentifier, newIdentifier, record.translatesTo())); } @@ -152,7 +149,7 @@ public class RuleIdentifierMigrator { } - public record Context(RuleFileBluePrint bluePrint, List records) { + public record Context(RuleFileBluePrint bluePrint) { } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java similarity index 73% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java index 8fdac0d7..53d1122c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import lombok.Getter; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java new file mode 100644 index 00000000..4e3acb5c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicDeclaration { + + @EqualsAndHashCode.Include + String declarationFullTypeName; + String body; + int line; + + + public static BasicDeclaration fromDeclarationDescription(AbstractClassTypeDeclarationDescr declaration, String rulesString) { + + String declarationFullTypeName = declaration.getFullTypeName(); + String body = rulesString.substring(declaration.getStartCharacter(), declaration.getEndCharacter()); + int line = declaration.getLine(); + + return new BasicDeclaration(declarationFullTypeName, body, line); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java new file mode 100644 index 00000000..1793d589 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java @@ -0,0 +1,36 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.FunctionDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicFunction { + + @EqualsAndHashCode.Include + String functionName; + String returnType; + String body; + int line; + + + public static BasicFunction fromFunctionDescr(FunctionDescr function, String rulesString) { + + String functionName = function.getName(); + String returnType = function.getReturnType(); + String body = rulesString.substring(function.getStartCharacter(), function.getEndCharacter()); + int line = function.getLine(); + + return new BasicFunction(functionName, returnType, body, line); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java similarity index 80% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java index 9d77bbb6..cc5c0fe6 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java @@ -1,8 +1,8 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; public record BasicRule(RuleIdentifier identifier, String name, String code) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java index a6cdb0ab..bb0e2d79 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java index 40468cc0..e70acc0f 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.LinkedList; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java similarity index 85% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 2b2ae91f..28b2663c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Collection; import java.util.Collections; @@ -10,7 +10,29 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -public record RuleFileBluePrint(String imports, String globals, String queries, List ruleClasses) { +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RuleFileBluePrint { + + String imports; + String globals; + String queries; + List ruleClasses; + List declarations; + List functions; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -24,7 +46,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, ruleClass.ruleUnits().remove(ruleUnit); } if (ruleClass.ruleUnits().isEmpty()) { - ruleClasses().remove(ruleClass); + this.ruleClasses.remove(ruleClass); } })); return wasRemoved.get(); @@ -41,7 +63,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRules() { - return ruleClasses().stream() + return this.ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -101,7 +123,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRuleIdentifiers() { - return ruleClasses().stream() + return ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -122,10 +144,16 @@ public record RuleFileBluePrint(String imports, String globals, String queries, } + public void setImports(String newImports) { + + this.imports = newImports; + } + + public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>()); - ruleClasses().stream() + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) .flatMap(ruleUnit -> ruleUnit.rules() diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java similarity index 97% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java index 569412e4..3fce64a7 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Arrays; import java.util.Objects; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java similarity index 95% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java index ffddc766..fcacf44d 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Map; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java similarity index 54% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java index ec23ad27..a2b84ba3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java similarity index 96% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java index 3d3ccac6..8efecd93 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.utils; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.utils; import java.io.File; import java.io.FileInputStream; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java new file mode 100644 index 00000000..32aaf4fc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -0,0 +1,169 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class RuleBuilderService { + + //todo: make this configurable + private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), + new RuleType("MAN"), + new RuleType("X"), + new RuleType("DICT"), + new RuleType("FA"), + new RuleType("LDS"))); + + + public RuleBuilderModel getRuleBuilderModel() { + + RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + + ruleBuilderModel.setWhenClauses(Collections.emptyList()); + ruleBuilderModel.setThenConditions(Collections.emptyList()); + + return ruleBuilderModel; + } + + + public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + log.info("Starting to remove system rules from ruleFile"); + removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); + log.info("Finished removing system rules for ruleFile"); + + return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); + } + + + public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { + + log.info("starting to merge user rules update with system rules"); + RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + mergedRuleFileBlueprint.getRuleClasses() + .forEach(ruleClass -> { + if (systemRules.stream() + .map(RuleType::name) + .toList().contains(ruleClass.ruleType().name())) { + throw new RuntimeException("No system rule updates allowed in user rule update."); + } + }); + removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getRuleClasses() + .stream() + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .flatMap(ruleUnit -> ruleUnit.rules() + .stream())) + .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); + mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() + .stream(), + mergedRuleFileBlueprint.getFunctions() + .stream()) + .toList()); + mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() + .stream(), + mergedRuleFileBlueprint.getDeclarations() + .stream()) + .toList()); + log.info("finished merging user rules update with system rules"); + RuleMergingResult mergingResult = RuleMergingResult.builder() + .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) + .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) + .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .build(); + + return mergingResult; + } + + + private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { + + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + removeAllRulesExceptSystemRules(ruleFileBluePrint); + ruleFileBluePrint.getRuleClasses() + .forEach(ruleClass -> { + if (!systemRuleNames.contains(ruleClass.ruleType().name())) { + throw new RuntimeException("there was an error removing all rules except system rules"); + } + }); + } + + + private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + + List rules = new ArrayList(); + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + + ruleFileBluePrint.getRuleClasses() + .stream() + .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) + .forEach(rule -> rules.add(rule)); + rules.forEach(ruleIdentifier -> { + ruleFileBluePrint.removeRule(ruleIdentifier); + }); + } + + + private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { + + removeSystemRules(ruleFileBluePrint); + for (RuleType systemRule : systemRules) { + List remainingSystemRules = new ArrayList(); + ruleFileBluePrint.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(remainingSystemRules::add)); + if (!remainingSystemRules.isEmpty()) { + throw new RuntimeException("There was an error removing the system rules from the file"); + } + } + } + + + private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { + + for (RuleType systemRule : systemRules) { + List rules = new ArrayList(); + ruleFileBluePrintExisting.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(rules::add)); + rules.forEach(ruleUnit -> { + ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); + }); + ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index 9b983752..e27361be 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -69,51 +69,64 @@ public class DroolsValidationService { private DroolsValidation buildCustomDroolsValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation(); + DroolsValidation customValidation = ruleCompilationResult.getDroolsValidation(); - addSyntaxDeprecatedWarnings(ruleFileBluePrint, customValidation); + addSyntaxDeprecatedWarnings(ruleCompilationResult, customValidation); - addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxErrorMessages(ruleFileType, ruleCompilationResult, customValidation); if (redactionServiceSettings.isRuleExecutionSecured()) { - addBlacklistErrorMessages(ruleFileBluePrint, customValidation); + addBlacklistErrorMessages(ruleCompilationResult, customValidation); } return customValidation; } - private void addSyntaxDeprecatedWarnings(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxDeprecatedWarnings(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { // find deprecated elements in the ruleFileBluePrint - DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); + DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleCompilationResult); if (warningMessageForImports != null) { customValidation.getDeprecatedWarnings().add(warningMessageForImports); } + customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleCompilationResult)); + } + + + private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleCompilationResult ruleCompilationResult) { + + return DroolsSyntaxDeprecatedWarnings.builder() + .message("global ComponentMappingService " + + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL + + "\n is missing from the rules, consider adding it, as it will be required in future versions!") + .line(ruleCompilationResult.getGlobalsLine()) + .column(0) + .build(); customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } - private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { + private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleCompilationResult ruleCompilationResult) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { - String imports = ruleFileBluePrint.getImports(); + String imports = ruleCompilationResult.getImports(); SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation(); List matches = classesSearchImplementation.getMatches(imports); if (!matches.isEmpty()) { String sb = "Following imports are deprecated: \n" + matches.stream() .map(m -> imports.substring(m.startIndex(), m.endIndex())) .collect(Collectors.joining("\n")); - return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build(); + return DroolsSyntaxDeprecatedWarnings.builder().line(ruleCompilationResult.getImportLine()).column(0).message(sb).build(); } } return null; } - private List getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { + private List getWarningsForDeprecatedRules(RuleCompilationResult ruleCompilationResult) { List warningMessages = new ArrayList<>(); @@ -122,7 +135,7 @@ public class DroolsValidationService { SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation(); Map deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap(); - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { List matches = methodsSearchImplementation.getMatches(basicRule.getCode()); @@ -146,32 +159,32 @@ public class DroolsValidationService { } - private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { - RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { - case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); - case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + RuleCompilationResult baseRuleCompilationResult = switch (ruleFileType) { + case ENTITY -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); + case COMPONENT -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseComponentRuleFileString()); }; - if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) { + if (!importsAreValid(baseRuleCompilationResult, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getImportLine()) + .line(ruleCompilationResult.getImportLine()) .column(0) - .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports())) + .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleCompilationResult.getImports())) .build()); } - if (!ruleFileBluePrint.getGlobals().contains(baseRuleFileBluePrint.getGlobals())) { + if (!ruleCompilationResult.getGlobals().contains(baseRuleCompilationResult.getGlobals())) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) - .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals())) + .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleCompilationResult.getGlobals())) .build()); } - baseRuleFileBluePrint.getQueries() + baseRuleCompilationResult.getQueries() .forEach(basicQuery -> { - if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { + if (!validateQueryIsPresent(basicQuery, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(basicQuery.getLine()) @@ -182,7 +195,7 @@ public class DroolsValidationService { }); if (ruleFileType.equals(RuleFileType.ENTITY)) { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; - if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { + if (!validateAgendaGroupIsPresent(ruleCompilationResult, requiredAgendaGroup)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(0) @@ -194,18 +207,18 @@ public class DroolsValidationService { } - private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) { + private boolean validateAgendaGroupIsPresent(RuleCompilationResult ruleCompilationResult, String agendaGroupName) { - return ruleFileBluePrint.streamAllRules() + return ruleCompilationResult.streamAllRules() .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName)); } - private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) { + private boolean importsAreValid(RuleCompilationResult baseRuleCompilationResult, RuleCompilationResult ruleCompilationResult) { // imports may shrink, but not add anything new! - Set baseImports = baseRuleFileBluePrint.getImportSplitByKeyword(); - Set imports = ruleFileBluePrint.getImportSplitByKeyword(); + Set baseImports = baseRuleCompilationResult.getImportSplitByKeyword(); + Set imports = ruleCompilationResult.getImportSplitByKeyword(); Set additionalImports = Sets.difference(imports, baseImports); @@ -214,15 +227,15 @@ public class DroolsValidationService { } - private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) { + private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleCompilationResult ruleCompilationResult) { - return ruleFileBluePrint.getQueries() + return ruleCompilationResult.getQueries() .stream() .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode())); } - private void addBlacklistErrorMessages(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addBlacklistErrorMessages(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { List blacklistErrorMessages = new ArrayList<>(); @@ -234,14 +247,14 @@ public class DroolsValidationService { // check also the imports DroolsBlacklistErrorMessage blacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, - ruleFileBluePrint.getImports(), - ruleFileBluePrint.getImportLine()); + ruleCompilationResult.getImports(), + ruleCompilationResult.getImportLine()); if (blacklistErrorMessage != null) { blacklistErrorMessages.add(blacklistErrorMessage); } // check the rules - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { DroolsBlacklistErrorMessage ruleBlacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java similarity index 53% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index cd6a953e..616cabde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -22,21 +22,48 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.RuleType; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @UtilityClass -public class RuleFileParser { +public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { + + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); + List allRules = new LinkedList<>(); + List allQueries = new LinkedList<>(); + PackageDescr packageDescr = new PackageDescr(); + String imports = ""; + String globals = ""; + List ruleClasses = buildRuleClasses(allRules); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + @SneakyThrows - public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); @@ -52,12 +79,15 @@ public class RuleFileParser { } } - String imports = ruleString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + String imports = ""; + if (!removedImports) { + imports = ruleString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -65,21 +95,81 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + + @SneakyThrows + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { + + return buildRuleCompilationResultFromRuleString(ruleString,false); + } + + + @SneakyThrows + public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { + + StringBuilder ruleStringBuilder = new StringBuilder(); + + // Append imports + ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); + + // Append globals + if (!bluePrint.getGlobals().isEmpty()) { + ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); + } + + // Append queries + for (BasicQuery query : bluePrint.getQueries()) { + ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); + } + + // Append rules + for (RuleClass ruleClass : bluePrint.getRuleClasses()) { + ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); + } + + // Return the final rule string + return ruleStringBuilder.toString().trim(); + } + + + private String buildQueryString(BasicQuery query) { + + return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; + } + + + private String buildRuleString(RuleClass ruleClass) { + + StringBuilder ruleBuilder = new StringBuilder(); + + // Use RuleType to distinguish between different rule types, if needed + ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); + + for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { + for (BasicRule rule : ruleUnit.rules()) { + // Assuming BasicRule has a method to retrieve the condition as a string + ruleBuilder.append(rule.getCode()).append("\n"); + } + } + ruleBuilder.append("end"); + + return ruleBuilder.toString(); } @@ -126,6 +216,7 @@ public class RuleFileParser { private List buildRuleClasses(List allRules) { + //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv diff --git a/redaction-service-v1/rules-management/src/main/resources/order_template.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/order_template.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java new file mode 100644 index 00000000..1996af83 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -0,0 +1,149 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RuleBuilderTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); + private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); + private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + + @Autowired + RuleBuilderService ruleBuilderService; + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import(LayoutParsingServiceProcessorConfiguration.class) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RuleBuilderTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + public void removeSystemRulesTest() { + + String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); + String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + + Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + + Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + } + + + @Test + public void mergeUserRulesUpdateTest() { + + RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); + String mergedRules = mergingResult.getMergedRules(); + RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); + ruleFileBluePrintUserRulesUpdate.getRuleClasses() + .forEach(ruleClass -> { + Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); + }); + + } + + + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getRuleClasses() + .stream() + .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) + .collect(Collectors.toList()).size() == 0; + } + + + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getImports().isEmpty(); + } + + + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getQueries().isEmpty(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java index cac2af47..2d95017d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java @@ -11,10 +11,10 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.core.io.ClassPathResource; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index b8fd95fe..2e42db62 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -22,11 +22,11 @@ import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; import lombok.SneakyThrows; @@ -235,11 +235,11 @@ class DroolsValidationServiceTest { if (droolsValidation.isCompiled()) { continue; } - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(rulesString); + RuleCompilationResult baseRuleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); - rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); - rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + rulesString = rulesString.replace(ruleCompilationResult.getImports(), baseRuleCompilationResult.getImports()); + rulesString = rulesString.replace(ruleCompilationResult.getGlobals(), baseRuleCompilationResult.getGlobals()); try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); @@ -460,9 +460,9 @@ class DroolsValidationServiceTest { end """; - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled()); + assertFalse(ruleCompilationResult.getDroolsValidation().isCompiled()); } } \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java similarity index 91% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java index f8d25b6a..fa77a92b 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,10 +27,10 @@ public class DroolsCompilationTest { @Test public void testValidateRuleSyntax() throws IOException { - URL rmURL = Resources.getResource("all_redact_manager_rules.drl"); + URL rmURL = Resources.getResource("rulesmanagement/all_redact_manager_rules.drl"); String rmRule = Resources.toString(rmURL, StandardCharsets.UTF_8); - URL dmURL = Resources.getResource("all_rules_documine.drl"); + URL dmURL = Resources.getResource("rulesmanagement/all_rules_documine.drl"); String dmRule = Resources.toString(dmURL, StandardCharsets.UTF_8); assertTrue(validateRuleSyntax(rmRule).isValid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java new file mode 100644 index 00000000..577f996a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; + +public class RuleFileBluePrintMergingTest { + + @Test + public void testBothRuleFilesCanBeMerged() { + + RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); + assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); + } + +} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java similarity index 93% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java index 2fc8aa7c..d9657a1c 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import java.io.File; import java.nio.file.Files; @@ -8,7 +8,7 @@ import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.migration.RuleFileMigrator; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java similarity index 94% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java index b6b85cef..ce84ebbf 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -19,10 +19,10 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/addNewRulesHere.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/addNewRulesHere.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl diff --git a/redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Basf-Demo/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Basf-Demo/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/dev/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_function_tests/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_function_tests/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_rule_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_rule_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_security_test/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_security_test/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/Manual_Redaction/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/Manual_Redaction/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/PII/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/PII/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/case_insensitive/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/case_insensitive/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/no_redaction_indicator/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/no_redaction_indicator/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/vertebrate_only/rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/qa/vertebrate_only/rules.txt diff --git a/redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl new file mode 100644 index 00000000..16ea73d2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl @@ -0,0 +1,50 @@ +package drools +//------------------------------------ Table extraction rules ------------------------------------ + +// Rule unit: TAB.0 +rule "TAB.0.0: Changed Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Changed Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +// Rule unit: TAB.6 +rule "TAB.6.1: Changed Targeted cell extraction (Experimental Stop date)" + when + $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2")) + $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList() + TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList() + $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList() + then + entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date")); + end + diff --git a/redaction-service-v1/rules-management/build.gradle.kts b/redaction-service-v1/rules-management/build.gradle.kts index 2aaea40c..b005ed7e 100644 --- a/redaction-service-v1/rules-management/build.gradle.kts +++ b/redaction-service-v1/rules-management/build.gradle.kts @@ -27,10 +27,10 @@ sourceSets { } dependencies { - implementation(project(":redaction-service-server-v1")) + implementation(project(mapOf("path" to ":redaction-service-server-v1"))) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") - + implementation(project(":redaction-service-server-v1")) implementation("com.github.javaparser:javaparser-core:3.25.3") implementation("org.drools:drools-drl-parser:8.41.0.Final") implementation("org.apache.commons:commons-csv:1.10.0") diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java index 7267bd20..6ab58cf3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java @@ -17,12 +17,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; @@ -72,7 +71,6 @@ public class Main { Set identifiers = switch (type) { case "list", "l" -> RuleIdentifier.fromListOfIdentifiersString(input); case "file", "f" -> RuleFileParser.parseRuleIdentifiersFromFile(input); - case "old", "o" -> OldRulesParser.translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(input), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid", cmd.getOptionValue("t"))); }; ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType); @@ -102,7 +100,6 @@ public class Main { String type = cmd.hasOption("t") ? cmd.getOptionValue("t") : "f"; Map> identifiersPerFile = switch (type) { case "file", "f" -> parseIdentifiersFromFiles(Path.of(cmd.getOptionValue("input"))); - case "old", "o" -> translateIdentifiersFromOldFiles(Path.of(cmd.getOptionValue("input")), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid in combination with the \"recursive\" flag", cmd.getOptionValue("t"))); }; identifiersPerFile.keySet() @@ -137,13 +134,6 @@ public class Main { } - @SneakyThrows - private static Map> translateIdentifiersFromOldFiles(Path inputDirectory, ApplicationType applicationType) { - - return RuleFileIO.streamAllRuleFilesInDirectory(inputDirectory) - .collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, applicationType))); - } - @SneakyThrows private static Map> parseIdentifiersFromFiles(Path inputDirectory) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java deleted file mode 100644 index af844a6d..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -public class RuleManagementResources { - - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_redact_manager_rules.drl"); - } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl"); - } - - - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers.txt"); - } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers_dm.txt"); - } - } - - - public static InputStream getTemplateInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("order_template.txt"); - } - - - public static InputStream getOldRulesCsvInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("old_rules_with_translations.csv"); - } - - - @SneakyThrows - public static String createTempOldRulesCsv(String formattedAsCsv) { - - File csvFile = File.createTempFile("old_rules_with_translations-", ".csv"); - try (var out = new FileOutputStream(csvFile)) { - out.write(formattedAsCsv.getBytes(StandardCharsets.UTF_8)); - } - return csvFile.toString(); - } - -} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java deleted file mode 100644 index ae0f3134..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java +++ /dev/null @@ -1,203 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; -import org.drools.drl.ast.descr.PackageDescr; -import org.drools.drl.ast.descr.RuleDescr; -import org.drools.drl.parser.DrlParser; -import org.kie.internal.builder.conf.LanguageLevelOption; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.OldRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; - -@UtilityClass -public class OldRulesParser { - - static List HEADERS = List.of("id", "old rule names", "old rule code", "translates to"); - - - @SneakyThrows - public void createSetOfTranslatedRuleIdentifiersForEachFileInTheCsv() { - - List allHeaders = List.of(); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - - } - - - public Set translateOldRulesFileToNewIdentifiers(File oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesFileToNewIdentifiers(oldRulesFile.toString(), applicationType); - } - - - public Set translateOldRulesFileToNewIdentifiers(String oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(oldRulesFile), applicationType); - } - - - public Set translateEscapedOldRulesStringToNewIdentifiers(String escapedOldRulesString, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.unescapeAndUnWrap(escapedOldRulesString), applicationType); - } - - - public Set translateOldRulesStringToNewIdentifiers(String oldRulesString, ApplicationType applicationType) { - - List oldRules = parseOldRules(oldRulesString); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - Map> translationPairs = new HashMap<>(); - for (OldRule oldRule : oldRules) { - List translatedIdentifiers = records.stream() - .filter(oldRulesCsvRecord -> oldRulesCsvRecord.code().equals(oldRule.code())) - .map(OldRulesCsvRecord::translatesTo) - .findAny() - .orElse(Collections.emptyList()); - translationPairs.put(oldRule, translatedIdentifiers); - } - boolean allTranslated = true; - for (OldRule oldRule : translationPairs.keySet()) { - if (translationPairs.get(oldRule).isEmpty()) { - allTranslated = false; - records.add(new OldRulesCsvRecord(records.size(), oldRule.code(), oldRule.name(), Collections.emptyList())); - System.out.printf( - "Rule %s has not been translated yet! %nIt has been added to the bottom of the old_rules_with_translations.csv file. To continue, translate the rule and enter its identifier in the csv, then try again!%n", - oldRule.name()); - } - } - if (!allTranslated) { - String formattedAsCsv = formatAsCsv(records); - String fileLocation = RuleManagementResources.createTempOldRulesCsv(formattedAsCsv); - System.out.printf("CSV File with updated values is located at %s%n", fileLocation); - throw new IllegalArgumentException("Non translated Rule found!"); - } - RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); - - translationPairs.forEach((key, value) -> System.out.printf("Rule %s will be translated to %s \n", - key.name(), - String.format("%s: %s", - value, - value.stream() - .map(bluePrint::findRuleByIdentifier) - .flatMap(Collection::stream) - .map(BasicRule::name) - .toList()))); - - return Stream.concat(// - Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),// - translationPairs.values() - .stream() - .flatMap(Collection::stream)) - .map(ruleIdentifier -> new RuleIdentifier(ruleIdentifier.type(), ruleIdentifier.unit(), null)) - .collect(Collectors.toSet()); - } - - - @SneakyThrows - public String formatAsCsv(List records) { - - StringWriter sw = new StringWriter(); - CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS.toArray(String[]::new)).build(); - try (CSVPrinter printer = new CSVPrinter(sw, csvFormat)) { - for (OldRulesCsvRecord record : records) { - printer.printRecord(Stream.of(record.id, RuleFileIO.escapeAndWrap(record.names), RuleFileIO.escapeAndWrap(record.code), record.translatesTo)); - } - } - return sw.toString(); - } - - - @SneakyThrows - private List parseOldRules(String oldRuleString) { - - DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); - PackageDescr packageDescr = parser.parse(false, oldRuleString); - List oldRules = new LinkedList<>(); - for (RuleDescr rule : packageDescr.getRules()) { - if (!rule.isRule()) { - continue; - } - oldRules.add(OldRule.fromRuleDescr(rule, oldRuleString)); - } - return oldRules; - } - - - @SneakyThrows - public List getOldRulesCsvRecords(InputStream allRulesCsvInputStream) { - - List parsedRecords = new LinkedList<>(); - try (Reader reader = new InputStreamReader(allRulesCsvInputStream)) { - Iterable records = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord().parse(reader); - for (CSVRecord record : records) { - String[] values = record.values(); - parsedRecords.add(new OldRulesCsvRecord(Long.parseLong(values[0]), - RuleFileIO.unescapeAndUnWrap(values[2]), - RuleFileIO.unescapeAndUnWrap(values[1]), - parseRuleIdentifiers(values[3]))); - } - } - return parsedRecords; - } - - - private List parseRuleIdentifiers(String value) { - - String cleanedValue = value; - if (cleanedValue.startsWith("[")) { - cleanedValue = cleanedValue.substring(1); - } - if (cleanedValue.endsWith("]")) { - cleanedValue = cleanedValue.substring(0, cleanedValue.length() - 1); - } - if (cleanedValue.isEmpty() || cleanedValue.isBlank()) { - return Collections.emptyList(); - } - List ruleIdentifiers = new LinkedList<>(); - for (String identifier : cleanedValue.split(", ")) { - ruleIdentifiers.add(RuleIdentifier.fromString(identifier)); - } - return ruleIdentifiers; - } - - - private List parseBooleanList(String[] values) { - - return Arrays.stream(values) - .map(Boolean::parseBoolean) - .toList(); - } - - - public record OldRulesCsvRecord(long id, String code, String names, List translatesTo) { - - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java deleted file mode 100644 index 559101ac..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; - -public class RuleFileBluePrintMergingTest { - - @Test - public void testBothRuleFilesCanBeMerged() { - - RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); - assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java deleted file mode 100644 index b78ab534..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -class OldRulesParserTest { - - @Test - @SneakyThrows - @Disabled - public void translateOldRulesToNewIdentifiersTest() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("efsa_regulation_rules.txt")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateEscapedOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @SneakyThrows - @Disabled - public void translateMoreOldRules() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("basf/demo/table_demo.drl")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @Disabled - public void printTranslationsTest() { - - List records = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")) - .forEach(type -> { - List rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type) - .orElseThrow().ruleUnits(); - rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records)); - }); - } - - - private void printOldRulesThatTranslatesToNewRule(RuleUnit ruleUnit, List records) { - - if (ruleUnit.rules().isEmpty()) { - System.out.println("Rule unit empty, skipping!\n"); - return; - } - RuleIdentifier identifier = ruleUnit.rules() - .get(0).identifier(); - RuleIdentifier unitIdentifier = new RuleIdentifier(identifier.type(), identifier.unit(), null); - String oldNames = records.stream() - .filter(r -> r.translatesTo() - .stream() - .anyMatch(i -> i.matches(unitIdentifier))) - .map(OldRulesParser.OldRulesCsvRecord::names) - .map(OldRulesParserTest::removeIdFromName) - .distinct() - .collect(Collectors.joining(", ")); - System.out.println(unitIdentifier.toRuleUnitString() + " " + ruleUnit.rules() - .stream() - .map(BasicRule::name) - .collect(Collectors.joining(", "))); - System.out.println("translate from"); - System.out.println(oldNames); - System.out.println(); - System.out.println(); - } - - - private static String removeIdFromName(String name) { - - String[] values = name.split(":"); - return String.join(":", Arrays.copyOfRange(values, 1, values.length)); - } - - - @Test - @Disabled - @SneakyThrows - public void findAllCustomerRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("pilot/EFSA_sanitisation_GFL_v1").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.RM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.RM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void findAllRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("business-logic/Syngenta_RSS").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.DM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.DM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void translateDossierTemplatesV2() { - -// String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/RED/dossier-templates-v2/"; -// Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), Files.walk(Path.of(dossierTemplatesRepo + "docu")), Files.walk(Path.of(dossierTemplatesRepo + "qa"))) - String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/"; - Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), - Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) - // - ) - .flatMap(Function.identity())// - .filter(path -> path.getFileName().toString().equals("rules.drl"))// - .map(Path::toFile)// - .forEach(this::translateOldRuleFile); - - } - - - @SneakyThrows - private void translateOldRuleFile(File oldRulesFile) { - - Set identifiers = OldRulesParser.translateOldRulesFileToNewIdentifiers(oldRulesFile, ApplicationType.RM); - String newRulesString = RuleFileFactory.createFileFromIdentifiers(identifiers, ApplicationType.RM); - String result = RuleFileIO.escapeAndWrap(newRulesString); - - try (var out = new FileOutputStream(oldRulesFile)) { - out.write(result.getBytes(StandardCharsets.UTF_8)); - } - } - -} \ No newline at end of file -- 2.47.2 From 23959262cfd159c5a8da7b7bac630be1f9840b3d Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 09:44:51 +0100 Subject: [PATCH 017/108] RED-9472: seperation of system rules code review changes --- .../controller/RuleBuilderController.java | 9 +- .../model/drools/RuleCompilationResult.java | 46 +---- .../RuleManagementResources.java | 19 +- .../factory/RuleFileParser.java | 15 +- .../models/RuleFileBluePrint.java | 28 +++ .../v1/server/service/RuleBuilderService.java | 91 ++------- .../drools/RuleCompilationResultParser.java | 83 +------- .../redaction/v1/server/RuleBuilderTest.java | 183 ++++++++---------- 8 files changed, 147 insertions(+), 327 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 8f3a51dd..aa215fc7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -45,7 +45,7 @@ public class RuleBuilderController implements RuleBuilderResource { RulesResponse rulesResponse = new RulesResponse(); String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); if (filteredRules.isEmpty()) { - throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + throw new AssertionError("There was an error when cleaning the rulefile of sytem rules"); } rulesResponse.setRules(filteredRules); return rulesResponse; @@ -53,13 +53,13 @@ public class RuleBuilderController implements RuleBuilderResource { @Override - public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); DroolsValidationResponse droolsValidationResponse; RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); if (mergingResult.getMergedRules().isEmpty()) { - throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); @@ -90,7 +90,8 @@ public class RuleBuilderController implements RuleBuilderResource { .toList()) .build(); if (!droolsValidation.isCompiled()) { - return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.UNPROCESSABLE_ENTITY); } else { // rulesUploadResponse.setRules(mergingResult.getMergedRules()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index e96aa697..3506a31f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -49,60 +49,16 @@ public final class RuleCompilationResult { public void dropRulesByIdentifier(RuleType ruleType) { - log.info("removing rule with identifier {}", ruleType.name()); - List rulesToBeRemoved = ruleClasses.stream() .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); ruleClasses.removeAll(rulesToBeRemoved); } - public void dropAllRulesExceptSystemRules(List systemRules) { - - log.info("removing all rules except the system rules"); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) - .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); - ruleClasses.removeAll(rulesToBeRemoved); - } - - - public void dropImports() { - - this.imports = ""; - } - - - public void dropQueries() { - - this.queries.clear(); - } - - - public int countRuleOccurences(RuleType ruleType) { - - log.info("counting occurences of files {}", ruleType.name()); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) - .toList(); - return rulesToBeRemoved.size(); - } - - - private boolean filterOutRule(RuleType ruleType, List filteredRules) { - - return !filteredRules.contains(ruleType.name()); - } - - public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replace("\n", "").split("import")) + return Arrays.stream(imports.replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java index 331b1e25..e423dde8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -1,34 +1,37 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement; +import java.io.IOException; import java.io.InputStream; +import org.drools.io.ClassPathResource; + import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; @SuppressWarnings("PMD") public class RuleManagementResources { - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + public static InputStream getAllRulesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + return new ClassPathResource("drools/all_redact_manager_rules.drl").getInputStream(); } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + return new ClassPathResource("drools/all_rules_documine.drl").getInputStream(); } - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers.txt").getInputStream(); } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers_dm.txt").getInputStream(); } } - public static InputStream getTemplateInputStream() { + public static InputStream getTemplateInputStream() throws IOException { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + return new ClassPathResource("rulesmanagement/order_template.txt").getInputStream(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index db0ece77..6354f93c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -4,7 +4,6 @@ import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -121,16 +120,12 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - if (rulesPerUnit.keySet() != null) { - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); - } else { - return new ArrayList<>(); - } + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 28b2663c..ca185b63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; @@ -53,6 +54,33 @@ public class RuleFileBluePrint { } + public boolean removeRuleClassByRuleIdentifier(RuleIdentifier ruleIdentifier) { + + AtomicBoolean wasRemoved = new AtomicBoolean(false); + + findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> { + List unitsToRemove = new ArrayList<>(); + + ruleClass.ruleUnits() + .forEach(ruleUnit -> { + boolean removed = ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier)); + if (removed) { + wasRemoved.set(true); + if (ruleUnit.rules().isEmpty()) { + unitsToRemove.add(ruleUnit); + } + } + }); + + ruleClass.ruleUnits().removeAll(unitsToRemove); + if (ruleClass.ruleUnits().isEmpty()) { + this.ruleClasses.remove(ruleClass); + } + }); + return wasRemoved.get(); + } + + public Optional findRuleClassByType(RuleType ruleType) { return ruleClasses.stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 32aaf4fc..d20e8337 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -3,9 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -16,8 +15,6 @@ import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFil import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.extern.slf4j.Slf4j; @@ -25,13 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - //todo: make this configurable - private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), - new RuleType("MAN"), - new RuleType("X"), - new RuleType("DICT"), - new RuleType("FA"), - new RuleType("LDS"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); public RuleBuilderModel getRuleBuilderModel() { @@ -48,28 +44,24 @@ public class RuleBuilderService { public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - log.info("Starting to remove system rules from ruleFile"); - removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); - log.info("Finished removing system rules for ruleFile"); - + removeSystemRules(ruleFileBluePrint); return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); } public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { - log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() - .map(RuleType::name) - .toList().contains(ruleClass.ruleType().name())) { - throw new RuntimeException("No system rule updates allowed in user rule update."); + .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { + log.warn("System rule in user rule update."); + //throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); ruleFileBluePrintExisting.getRuleClasses() .stream() .flatMap(ruleClass -> ruleClass.ruleUnits() @@ -89,7 +81,6 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - log.info("finished merging user rules update with system rules"); RuleMergingResult mergingResult = RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) @@ -100,69 +91,17 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { - - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); - removeAllRulesExceptSystemRules(ruleFileBluePrint); - ruleFileBluePrint.getRuleClasses() - .forEach(ruleClass -> { - if (!systemRuleNames.contains(ruleClass.ruleType().name())) { - throw new RuntimeException("there was an error removing all rules except system rules"); - } - }); - } - - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - List rules = new ArrayList(); - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); - ruleFileBluePrint.getRuleClasses() - .stream() - .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) - .forEach(rule -> rules.add(rule)); - rules.forEach(ruleIdentifier -> { - ruleFileBluePrint.removeRule(ruleIdentifier); - }); - } - - - private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { - - removeSystemRules(ruleFileBluePrint); - for (RuleType systemRule : systemRules) { - List remainingSystemRules = new ArrayList(); - ruleFileBluePrint.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(remainingSystemRules::add)); - if (!remainingSystemRules.isEmpty()) { - throw new RuntimeException("There was an error removing the system rules from the file"); - } - } } private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { - for (RuleType systemRule : systemRules) { - List rules = new ArrayList(); - ruleFileBluePrintExisting.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(rules::add)); - rules.forEach(ruleUnit -> { - ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); - }); - ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + for (RuleIdentifier systemRule : systemRules) { + ruleFileBluePrintExisting.removeRuleClassByRuleIdentifier(systemRule); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 616cabde..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -35,33 +35,6 @@ public class RuleCompilationResultParser { "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { - - DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); - List allRules = new LinkedList<>(); - List allQueries = new LinkedList<>(); - PackageDescr packageDescr = new PackageDescr(); - String imports = ""; - String globals = ""; - List ruleClasses = buildRuleClasses(allRules); - return new RuleCompilationResult(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); - } - - @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { @@ -116,60 +89,7 @@ public class RuleCompilationResultParser { @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { - return buildRuleCompilationResultFromRuleString(ruleString,false); - } - - - @SneakyThrows - public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { - - StringBuilder ruleStringBuilder = new StringBuilder(); - - // Append imports - ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); - - // Append globals - if (!bluePrint.getGlobals().isEmpty()) { - ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); - } - - // Append queries - for (BasicQuery query : bluePrint.getQueries()) { - ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); - } - - // Append rules - for (RuleClass ruleClass : bluePrint.getRuleClasses()) { - ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); - } - - // Return the final rule string - return ruleStringBuilder.toString().trim(); - } - - - private String buildQueryString(BasicQuery query) { - - return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; - } - - - private String buildRuleString(RuleClass ruleClass) { - - StringBuilder ruleBuilder = new StringBuilder(); - - // Use RuleType to distinguish between different rule types, if needed - ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); - - for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { - for (BasicRule rule : ruleUnit.rules()) { - // Assuming BasicRule has a method to retrieve the condition as a string - ruleBuilder.append(rule.getCode()).append("\n"); - } - } - ruleBuilder.append("end"); - - return ruleBuilder.toString(); + return buildRuleCompilationResultFromRuleString(ruleString, false); } @@ -216,7 +136,6 @@ public class RuleCompilationResultParser { private List buildRuleClasses(List allRules) { - //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 1996af83..181e108a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,149 +1,128 @@ package com.iqser.red.service.redaction.v1.server; +import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; import static org.mockito.Mockito.when; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; -import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; -import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class RuleBuilderTest extends AbstractRedactionIntegrationTest { +import org.junit.jupiter.api.Assertions; +import lombok.SneakyThrows; - private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); - private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); - private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); - - @Autowired - RuleBuilderService ruleBuilderService; - - @Configuration - @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) - @Import(LayoutParsingServiceProcessorConfiguration.class) - @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) - public static class RuleBuilderTestConfiguration { - - @Bean - @Primary - public StorageService inmemoryStorage() { - - return new FileSystemBackedStorageService(ObjectMapperFactory.create()); - } - - } +class RuleBuilderTest { + private RuleBuilderService ruleBuilderService; + private final List systemRules = new ArrayList<>(Arrays.asList( + RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*") + )); @BeforeEach - public void stubClients() { - + void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); - - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); - - loadDictionaryForTest(); - loadTypeForTest(); - loadNerForTest(); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); - - when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); - - mockDictionaryCalls(null); - - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } - @Test - public void removeSystemRulesTest() { + @SneakyThrows + void removeSystemRulesTest() throws IOException { + URL url = getClass().getClassLoader().getResource("drools"); + if (url == null) { + throw new RuntimeException("Drools directory not found in resources"); + } - String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); - String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); - RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + Path path = Paths.get(url.toURI()); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); - Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); - - Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } + private void testThatSystemRulesAreRemoved(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + true + ); - @Test - public void mergeUserRulesUpdateTest() { + RuleCompilationResult ruleCompilationResultWithImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); - RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); - String mergedRules = mergingResult.getMergedRules(); - RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); - ruleFileBluePrintUserRulesUpdate.getRuleClasses() - .forEach(ruleClass -> { - Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); - }); + Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports + ); + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + + Assertions.assertEquals( + RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + } private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getRuleClasses() - .stream() - .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) - .collect(Collectors.toList()).size() == 0; + .stream() + .map(RuleClass::ruleType) + .noneMatch(ruleType -> systemRules.stream() + .map(RuleIdentifier::type) + .anyMatch(type -> ruleType.name().equals(type.name()))); } - private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getImports().isEmpty(); } - private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getQueries().isEmpty(); } - -} +} \ No newline at end of file -- 2.47.2 From 463154a4ee4c3f08a6a038c87ba4cbf261d970d5 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 018/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 169500183aba3450a4e3dd015539197e0f87931e Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 019/108] RED-9472: seperation of system rules pmd fix --- .../v1/server/service/RuleBuilderService.java | 20 +++++++++---------- .../drools/RuleCompilationResultParser.java | 8 ++------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index d20e8337..2c9c9f7e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -58,7 +58,7 @@ public class RuleBuilderService { if (systemRules.stream() .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { log.warn("System rule in user rule update."); - //throw new RuntimeException("No system rule updates allowed in user rule update."); + throw new RuntimeException("No system rule updates allowed in user rule update."); } }); removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); @@ -81,19 +81,17 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - RuleMergingResult mergingResult = RuleMergingResult.builder() + return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) .build(); - - return mergingResult; } private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From e47dc30770f6495a3eea85a6e3dcccf4b6f36ba4 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:26:32 +0100 Subject: [PATCH 020/108] RED-9472: seperation of system rules pmd fix --- .../redaction/v1/server/RuleBuilderTest.java | 83 ++++++++----------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 181e108a..31d61e1e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server; -import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URL; import java.nio.file.Path; @@ -13,8 +8,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; @@ -27,35 +24,32 @@ import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.knecon.fforesight.tenantcommons.TenantContext; -import org.junit.jupiter.api.Assertions; import lombok.SneakyThrows; class RuleBuilderTest { private RuleBuilderService ruleBuilderService; - private final List systemRules = new ArrayList<>(Arrays.asList( - RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*") - )); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); + @BeforeEach void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); } + @Test @SneakyThrows void removeSystemRulesTest() throws IOException { - URL url = getClass().getClassLoader().getResource("drools"); - if (url == null) { - throw new RuntimeException("Drools directory not found in resources"); - } + URL url = Thread.currentThread().getContextClassLoader().getResource("drools"); Path path = Paths.get(url.toURI()); RuleFileIO.streamAllRuleFilesInDirectory(path) .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); @@ -64,20 +58,14 @@ class RuleBuilderTest { .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } - private void testThatSystemRulesAreRemoved(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - true - ); - RuleCompilationResult ruleCompilationResultWithImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + private void testThatSystemRulesAreRemoved(Path path) { + + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), true); + + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); @@ -88,28 +76,22 @@ class RuleBuilderTest { Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - cleanedRulesWithImports - ); + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports); - Assertions.assertEquals( - RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(cleanedRuleFileMergedWithSystemRules.getMergedRules()); + + Assertions.assertEquals(RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), cleanedRuleFileMergedWithSystemRules.getMergedRules()); } + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getRuleClasses() .stream() .map(RuleClass::ruleType) @@ -118,11 +100,16 @@ class RuleBuilderTest { .anyMatch(type -> ruleType.name().equals(type.name()))); } + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getImports().isEmpty(); } + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getQueries().isEmpty(); } + } \ No newline at end of file -- 2.47.2 From 82bfa1b5e9af8bf208c24a3b1e4ae600508d1464 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 15 Nov 2024 09:45:56 +0100 Subject: [PATCH 021/108] RED-9472: seperation of system rules review changes --- .../models/RuleFileBluePrint.java | 2 +- .../v1/server/service/RuleBuilderService.java | 46 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index ca185b63..533492e1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -180,7 +180,7 @@ public class RuleFileBluePrint { public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), declarations, functions); ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 2c9c9f7e..c346429c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -1,10 +1,10 @@ package com.iqser.red.service.redaction.v1.server.service; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), - RuleIdentifier.fromName("MAN"), - RuleIdentifier.fromName("X"), - RuleIdentifier.fromName("DICT"), - RuleIdentifier.fromName("FA"), - RuleIdentifier.fromName("LDS"))); + private final HashSet systemRules = new HashSet<>(Set.of(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -52,7 +52,7 @@ public class RuleBuilderService { public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() @@ -61,25 +61,30 @@ public class RuleBuilderService { throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - ruleFileBluePrintExisting.getRuleClasses() - .stream() - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .flatMap(ruleUnit -> ruleUnit.rules() - .stream())) + ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getAllRules() .forEach(mergedRuleFileBlueprint::addRule); - mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); - mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setImports(uniqueImports.stream() + .collect(Collectors.joining(""))); + Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setGlobals(uniqueGlobals.stream() + .collect(Collectors.joining(""))); mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() .stream(), mergedRuleFileBlueprint.getFunctions() .stream()) + .distinct() .toList()); mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() .stream(), mergedRuleFileBlueprint.getDeclarations() .stream()) + .distinct() .toList()); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) @@ -89,10 +94,9 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + private RuleFileBluePrint removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + return ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } -- 2.47.2 From e5d00de2efe0ae39160da6e1a4f692eb6ce0513c Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 022/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 6b7cf3a4bdca4e6c9a7689f00f93741908d44004 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 023/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 6208017db172cf233ab8d00eeb2044e6188c382e Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 09:04:46 +0100 Subject: [PATCH 024/108] RED-9472: seperation of system rules added document dependency --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 277f9fde..cbefd926 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -38,7 +38,7 @@ configurations.all { } dependencies { - + implementation(project(":document")) implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") -- 2.47.2 From f746729851fe253d91a1d7b354cf9718f31ae937 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 13:02:03 +0100 Subject: [PATCH 025/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index cbefd926..594dc06a 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.585.0-RED9472.1" +val persistenceServiceVersion = "2.627.0-RED9472.2" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" @@ -45,6 +45,7 @@ dependencies { { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } + implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}") implementation("com.iqser.red.commons:spring-commons:6.2.0") @@ -62,6 +63,7 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") implementation("org.ahocorasick:ahocorasick:0.6.3") + implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") -- 2.47.2 From 76120908e8c5e19459980dcdfd8fb4b1368c4c70 Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 08:52:32 +0100 Subject: [PATCH 026/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 594dc06a..45858e36 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.627.0-RED9472.2" +val persistenceServiceVersion = "2.628.0-RED9472.0" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" -- 2.47.2 From 57408df41365fa076b12ff758f098cf86d3187b7 Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 09:55:55 +0100 Subject: [PATCH 027/108] RED-9472: seperation of system rules added updated persistence service version --- redaction-service-v1/redaction-service-api-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index ccff795e..811fa166 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.585.0-RED9472.0" +val persistenceServiceVersion = "2.628.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") -- 2.47.2 From f0129955df51ff08b500dad4c4b2a4f14dff2d08 Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 10:04:51 +0100 Subject: [PATCH 028/108] RED-9472: seperation of system rules added updated service version for fixing checkstyle errors --- .../redaction-service-server-v1/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 45858e36..447e61ca 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -53,7 +53,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.50.0") - implementation("com.knecon.fforesight:tenant-commons:0.30.0") + implementation("com.knecon.fforesight:tenant-commons:0.31.0") implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } @@ -64,6 +64,8 @@ dependencies { implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") implementation("org.ahocorasick:ahocorasick:0.6.3") implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") + implementation("com.github.roklenarcic:aho-corasick:1.2") + implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") -- 2.47.2 From 5f97d7acb36da006eddc127936437716cd1a0f11 Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 10:18:10 +0100 Subject: [PATCH 029/108] RED-9472: seperation of system rules added updated service version for fixing checkstyle errors --- .../v1/server/service/drools/DroolsValidationService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index b42326c5..b1dfa5f0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -105,7 +105,6 @@ public class DroolsValidationService { .line(ruleCompilationResult.getGlobalsLine()) .column(0) .build(); - customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } -- 2.47.2 From 35a572a3c5e73ba7337c82d1fca40c737f2be6fb Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 11:59:58 +0100 Subject: [PATCH 030/108] RED-9472: seperation of system rules added updated service versions --- .../build.gradle.kts | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 447e61ca..29b5aec8 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -12,12 +12,12 @@ plugins { description = "redaction-service-server-v1" -val layoutParserVersion = "0.181.0" +val layoutParserVersion = "0.191.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" val persistenceServiceVersion = "2.628.0-RED9472.0" -val llmServiceVersion = "1.11.0" +val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" @@ -31,21 +31,15 @@ configurations { } } -configurations.all { - resolutionStrategy { - force("com.google.protobuf:protobuf-java:4.27.1") - } -} - dependencies { - implementation(project(":document")) + implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } + implementation(project(":document")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } - implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}") implementation("com.iqser.red.commons:spring-commons:6.2.0") @@ -62,7 +56,7 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") - implementation("org.ahocorasick:ahocorasick:0.6.3") + implementation("org.ahocorasick:ahocorasick:0.9.0") implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") implementation("com.github.roklenarcic:aho-corasick:1.2") @@ -71,7 +65,6 @@ dependencies { implementation("org.drools:drools-engine:${droolsVersion}") implementation("org.drools:drools-mvel:${droolsVersion}") implementation("org.kie:kie-spring:7.74.1.Final") - implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.locationtech.jts:jts-core:1.19.0") @@ -87,7 +80,6 @@ dependencies { implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") - api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") @@ -134,6 +126,7 @@ tasks.named("bootBuildImage") { "BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5" ) + environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE! if (project.hasProperty("buildbootDockerHostNetwork")) { @@ -188,10 +181,15 @@ tasks.register("generateJavaDoc", Javadoc::class) { dependsOn("compileJava") dependsOn("delombok") classpath = project.sourceSets["main"].runtimeClasspath - source = fileTree("${buildDir}/generated/sources/delombok/java/main") { + val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { include(droolsImports) } - destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "") + val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { + include(droolsImports) + } + source = documentFiles + mainFiles + + setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "")) options.memberLevel = JavadocMemberLevel.PUBLIC (options as StandardJavadocDocletOptions).apply { -- 2.47.2 From 47eda6290fde3aaaeafd184f2a78050f0cf2ac3a Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Dec 2024 13:35:51 +0100 Subject: [PATCH 031/108] RED-9472: seperation of system rules --- .../service/redaction/v1/server/service/RuleBuilderService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index c346429c..54670ba4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -86,6 +86,7 @@ public class RuleBuilderService { .stream()) .distinct() .toList()); + log.info("merged rules: {}", RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) -- 2.47.2 From 4ed0f51cccb51a748a23971e284e1bb0ca7afc29 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Sep 2024 10:24:59 +0200 Subject: [PATCH 032/108] RED-9472: seperation of system rules added endpoints for seperating and readding system rules working on service RED-9472: seperation of system rules refactored endpoints. now removing system rules on download merging user rules with systemrules on upload RED-9472: seperation of system rules RED-9472: seperation of system rules RED-9472: seperation of system rules refactored two have a rulefileblueprint and a rulefileblueprint for validationresult changed the flow of merging and seperating system and user rules added functions and declarations to rule file parsing RED-9472: seperation of system rules RED-9472: seperation of system rules introduced tests working on fixing bug with merging rules RED-9472: seperation of system rules introduced tests RED-9472: seperation of system rules fixed circular depdency RED-9472: seperation of system rules updated persistence service version RED-9472: seperation of system rules refactored rules refactored pcackages RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed some sonarqube errors RED-9472: seperation of system rules fixed pmd error RED-9472: seperation of system rules added deploy stage to gitlab pipeline RED-9472: seperation of system rules RED-9472: seperation of system rules fixed a mistake RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse --- .../redaction-service-api-v1/build.gradle.kts | 3 +- .../v1/resources/RuleBuilderResource.java | 17 +- .../build.gradle.kts | 28 ++- .../controller/RuleBuilderController.java | 96 ++++++++- .../v1/server/model/RuleMergingResult.java | 24 +++ ...ePrint.java => RuleCompilationResult.java} | 67 +++++- .../RuleManagementResources.java | 34 +++ .../DefaultRuleIdentifiersFactory.java | 8 +- .../factory/RuleFileFactory.java | 51 +++-- .../factory/RuleFileParser.java | 82 +++++-- .../migration/RuleFileMigrator.java | 15 +- .../migration/RuleIdentifierMigrator.java | 45 ++-- .../models/ApplicationType.java | 2 +- .../models/BasicDeclaration.java | 34 +++ .../rulesmanagement/models/BasicFunction.java | 36 ++++ .../rulesmanagement}/models/BasicRule.java | 4 +- .../rulesmanagement}/models/OldRule.java | 2 +- .../rulesmanagement}/models/RuleClass.java | 2 +- .../models/RuleFileBluePrint.java | 42 +++- .../models/RuleIdentifier.java | 2 +- .../rulesmanagement}/models/RuleType.java | 2 +- .../rulesmanagement}/models/RuleUnit.java | 2 +- .../rulesmanagement}/utils/RuleFileIO.java | 2 +- .../v1/server/service/RuleBuilderService.java | 169 +++++++++++++++ .../drools/DroolsValidationService.java | 85 ++++---- ....java => RuleCompilationResultParser.java} | 139 +++++++++--- .../all_redact_manager_rules.drl | 0 .../rulesmanagement}/all_rules_documine.drl | 0 .../default_rule_identifiers.txt | 0 .../default_rule_identifiers_dm.txt | 0 .../old_rules_with_translations.csv | 0 .../rulesmanagement}/order_template.txt | 0 .../redaction/v1/server/RuleBuilderTest.java | 149 +++++++++++++ .../services/DroolsUpToDateTest.java | 8 +- .../services/DroolsValidationServiceTest.java | 16 +- .../DroolsCompilationTest.java | 6 +- .../RuleFileBluePrintMergingTest.java | 20 ++ .../RuleFileMigrationTest.java | 4 +- .../factory/RuleFileFactoryTest.java | 10 +- .../EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rulesmanagement/addNewRulesHere.drl | 0 .../rulesmanagement}/demo/table_demo.drl | 0 .../rulesmanagement/dev/Flora/rules.txt | 0 .../docu/EFSA_Regulation_2021/rules.txt | 0 .../docu/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../docu/Manual_Redaction/rules.txt | 0 .../rulesmanagement/efsa_regulation_rules.txt | 0 .../userrules/user_rule_update.drl | 50 +++++ .../rules-management/build.gradle.kts | 4 +- .../utility/rules/management/Main.java | 20 +- .../management/RuleManagementResources.java | 56 ----- .../translation/OldRulesParser.java | 203 ------------------ .../RuleFileBluePrintMergingTest.java | 20 -- .../translation/OldRulesParserTest.java | 176 --------------- 55 files changed, 1063 insertions(+), 672 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/{RuleFileBluePrint.java => RuleCompilationResult.java} (61%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/DefaultRuleIdentifiersFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileParser.java (50%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleFileMigrator.java (77%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleIdentifierMigrator.java (76%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/ApplicationType.java (73%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/BasicRule.java (80%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/OldRule.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleClass.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleFileBluePrint.java (85%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleIdentifier.java (97%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleType.java (95%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleUnit.java (54%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/utils/RuleFileIO.java (96%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/{RuleFileParser.java => RuleCompilationResultParser.java} (53%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_redact_manager_rules.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_rules_documine.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers_dm.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/old_rules_with_translations.csv (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/order_template.txt (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/DroolsCompilationTest.java (91%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/RuleFileMigrationTest.java (93%) rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactoryTest.java (94%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/demo/table_demo.drl (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 87ec3822..dc942464 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,11 +4,12 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.631.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" dependencies { implementation("org.springframework:spring-web:6.0.12") implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + api("com.knecon.fforesight:swagger-commons:0.7.0") } publishing { diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index 3864ba9b..4683fd5f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,13 +1,26 @@ package com.iqser.red.service.redaction.v1.resources; -import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; - import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; public interface RuleBuilderResource { @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE) RuleBuilderModel getRuleBuilderModel(); + + @PostMapping(value = "/internal-api/rules/user-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithoutSystemRules(@RequestBody SystemRulesSeperationRequest systemRulesSeperationRequest); + + + @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + ResponseEntity mergeUserUpdateRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); + } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 262655fd..f3d723ae 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.641.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" @@ -31,10 +31,15 @@ configurations { } } +configurations.all { + resolutionStrategy { + force("com.google.protobuf:protobuf-java:4.27.1") + } +} + dependencies { implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } - implementation(project(":document")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { @@ -47,7 +52,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.50.0") - implementation("com.knecon.fforesight:tenant-commons:0.31.0") + implementation("com.knecon.fforesight:tenant-commons:0.30.0") implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } @@ -56,14 +61,13 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") - implementation("org.ahocorasick:ahocorasick:0.9.0") - implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") - implementation("com.github.roklenarcic:aho-corasick:1.2") + implementation("org.ahocorasick:ahocorasick:0.6.3") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") implementation("org.drools:drools-mvel:${droolsVersion}") implementation("org.kie:kie-spring:7.74.1.Final") + implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.locationtech.jts:jts-core:1.19.0") @@ -82,6 +86,7 @@ dependencies { implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") + api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") @@ -132,7 +137,6 @@ tasks.named("bootBuildImage") { "BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5" ) - environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE! if (project.hasProperty("buildbootDockerHostNetwork")) { @@ -187,19 +191,13 @@ tasks.register("generateJavaDoc", Javadoc::class) { dependsOn("compileJava") dependsOn("delombok") classpath = project.sourceSets["main"].runtimeClasspath - val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { + source = fileTree("${buildDir}/generated/sources/delombok/java/main") { include(droolsImports) } - val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { - include(droolsImports) - } - source = documentFiles + mainFiles - - setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "")) + destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "") options.memberLevel = JavadocMemberLevel.PUBLIC (options as StandardJavadocDocletOptions).apply { title = "API Documentation for Redaction Service ${project.version}" } } - diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 306dccbe..8f3a51dd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,27 +1,111 @@ package com.iqser.red.service.redaction.v1.server.controller; -import java.util.Collections; - +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleBlacklistErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxWarningMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @RestController +@Slf4j @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { + private final RuleBuilderService ruleBuilderService; + private final DroolsValidationService droolsValidationService; + + @Override public RuleBuilderModel getRuleBuilderModel() { - RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + return this.ruleBuilderService.getRuleBuilderModel(); + } - ruleBuilderModel.setWhenClauses(Collections.emptyList()); - ruleBuilderModel.setThenConditions(Collections.emptyList()); - return ruleBuilderModel; + @Override + public RulesResponse getRuleFileWithoutSystemRules(SystemRulesSeperationRequest systemRulesSeperationRequest) { + + RulesResponse rulesResponse = new RulesResponse(); + String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); + if (filteredRules.isEmpty()) { + throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + } + rulesResponse.setRules(filteredRules); + return rulesResponse; + } + + + @Override + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + + RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); + DroolsValidationResponse droolsValidationResponse; + RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + if (mergingResult.getMergedRules().isEmpty()) { + throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + } + try { + var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + droolsValidationResponse = DroolsValidationResponse.builder() + .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() + .stream() + .map(droolsSyntaxErrorMessage -> new RuleSyntaxErrorMessage(droolsSyntaxErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxErrorMessage.getColumn(), + droolsSyntaxErrorMessage.getMessage())) + .toList()) + .deprecatedWarnings(droolsValidation.getDeprecatedWarnings() + .stream() + .map(droolsSyntaxDeprecatedWarnings -> new RuleSyntaxWarningMessage(droolsSyntaxDeprecatedWarnings.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxDeprecatedWarnings.getColumn(), + droolsSyntaxDeprecatedWarnings.getMessage())) + .toList()) + .blacklistErrorMessages(droolsValidation.getBlacklistErrorMessages() + .stream() + .map(droolsBlacklistErrorMessage -> new RuleBlacklistErrorMessage(droolsBlacklistErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsBlacklistErrorMessage.getColumn(), + droolsBlacklistErrorMessage.getMessage())) + .toList()) + .build(); + if (!droolsValidation.isCompiled()) { + return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + } else { + // + rulesUploadResponse.setRules(mergingResult.getMergedRules()); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + } + } catch (Exception e) { + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + } + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); + } + + + private enum RuleFileType { + ENTITY, + COMPONENT } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java new file mode 100644 index 00000000..b2cb1b27 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Schema(description = "Object containing a string of Drools rules.") +public class RuleMergingResult { + + @Schema(description = "The merged rules.") + private String mergedRules; + + @Schema(description = "the length of added imports from sytemRules") + private int addedImportsOffset; + @Schema(description = "the length of added globals from sytemRules") + private int addedGlobalsOffset; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java similarity index 61% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index 3aa1cdef..e96aa697 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -20,13 +20,15 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Builder @NoArgsConstructor @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public final class RuleFileBluePrint { +public final class RuleCompilationResult { String imports; int importLine; @@ -45,9 +47,62 @@ public final class RuleFileBluePrint { } + public void dropRulesByIdentifier(RuleType ruleType) { + + log.info("removing rule with identifier {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropAllRulesExceptSystemRules(List systemRules) { + + log.info("removing all rules except the system rules"); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropImports() { + + this.imports = ""; + } + + + public void dropQueries() { + + this.queries.clear(); + } + + + public int countRuleOccurences(RuleType ruleType) { + + log.info("counting occurences of files {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + return rulesToBeRemoved.size(); + } + + + private boolean filterOutRule(RuleType ruleType, List filteredRules) { + + return !filteredRules.contains(ruleType.name()); + } + + public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replaceAll("\n", "").split("import")) + return Arrays.stream(imports.replace("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } @@ -76,7 +131,7 @@ public final class RuleFileBluePrint { public List getAllRuleIdentifiers() { return streamAllRules().map(BasicRule::getIdentifier) - .collect(Collectors.toList()); + .toList(); } @@ -96,4 +151,10 @@ public final class RuleFileBluePrint { return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']'; } + + public void addRuleClass(RuleClass ruleClass) { + + this.ruleClasses.add(ruleClass); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java new file mode 100644 index 00000000..331b1e25 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import java.io.InputStream; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; + +@SuppressWarnings("PMD") +public class RuleManagementResources { + + public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + } + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + } + + + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + } else { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + } + } + + + public static InputStream getTemplateInputStream() { + + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java index 2a8de641..c83f7f04 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java @@ -1,12 +1,12 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java index 039664bf..f6332332 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.Collections.emptySet; @@ -9,13 +9,13 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -48,17 +48,17 @@ public class RuleFileFactory { } RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers); - return buildRuleString(filteredBluePrint); + return buildRuleString(filteredBluePrint, false, false); } @SneakyThrows - public String buildRuleString(RuleFileBluePrint bluePrint) { + public String buildRuleString(RuleFileBluePrint bluePrint, boolean dropImports, boolean dropQueries) { try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) { String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8); List templateRuleOrder = parseRuleOrder(template); - return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder); + return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder, dropImports, dropQueries); } } @@ -72,23 +72,35 @@ public class RuleFileFactory { } - private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder) { + private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder, boolean dropImports, boolean dropQueries) { - Set additionalRuleTypes = bluePrint.ruleClasses() + Set additionalRuleTypes = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleType) .filter(ruleType -> !ruleOrder.contains(ruleType)) .collect(Collectors.toSet()); StringBuilder sb = new StringBuilder(); - sb.append(bluePrint.imports()); + if (!dropImports) { + sb.append(bluePrint.getImports()); + sb.append("\n\n"); + } else { + //todo: this is hacked to enable compiling the rules without imports + sb.append("package drools"); + sb.append("\n\n"); + } + sb.append(bluePrint.getGlobals()); sb.append("\n\n"); - sb.append(bluePrint.globals()); + sb.append("//------------------------------------ declarations ------------------------------------"); sb.append("\n\n"); - sb.append("//------------------------------------ queries ------------------------------------"); - sb.append("\n\n"); - sb.append(bluePrint.queries()); + sb.append(bluePrint.getDeclarations().isEmpty() ? "" : bluePrint.getDeclarations()); sb.append("\n\n"); + if (!dropQueries) { + sb.append("//------------------------------------ queries ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getQueries()); + sb.append("\n\n"); + } for (RuleType ruleBlockType : ruleOrder) { if (ruleBlockType.isWildCard()) { additionalRuleTypes.stream() @@ -101,6 +113,9 @@ public class RuleFileFactory { } writeRuleClass(bluePrint, ruleBlockType, sb); } + sb.append("//------------------------------------ functions ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getFunctions().isEmpty() ? "" : bluePrint.getFunctions()); return sb.toString().trim() + "\n"; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java similarity index 50% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index 3e17b0db..db0ece77 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -1,30 +1,35 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; +import org.drools.drl.ast.descr.FunctionDescr; import org.drools.drl.ast.descr.ImportDescr; import org.drools.drl.ast.descr.PackageDescr; import org.drools.drl.ast.descr.RuleDescr; import org.drools.drl.parser.DrlParser; import org.kie.internal.builder.conf.LanguageLevelOption; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicDeclaration; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicFunction; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -45,10 +50,19 @@ public class RuleFileParser { @SneakyThrows public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString) { + return buildBluePrintFromRulesString(rulesString, false); + } + + + @SneakyThrows + public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString, boolean removeImports) { + DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); PackageDescr packageDescr = parser.parse(false, rulesString); StringBuilder queryBuilder = new StringBuilder(); List allRules = new LinkedList<>(); + List functions = new LinkedList<>(); + List declarations = new LinkedList<>(); for (RuleDescr rule : packageDescr.getRules()) { if (rule.isQuery()) { queryBuilder.append(rulesString, rule.getStartCharacter(), rule.getEndCharacter()); @@ -57,12 +71,30 @@ public class RuleFileParser { } allRules.add(BasicRule.fromRuleDescr(rule, rulesString)); } - String imports = rulesString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + for (FunctionDescr function : packageDescr.getFunctions()) { + functions.add(BasicFunction.fromFunctionDescr(function, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getEnumDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getClassAndEnumDeclarationDescrs()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + String imports = ""; + if (!removeImports && !packageDescr.getImports().isEmpty()) { + imports = rulesString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -70,7 +102,7 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses); + return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses, declarations, functions); } @@ -89,11 +121,17 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); + if (rulesPerUnit.keySet() != null) { + + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); + } else { + return new ArrayList<>(); + } + } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java similarity index 77% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java index c002fed3..afbdff26 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java @@ -1,15 +1,16 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.File; import java.io.FileOutputStream; import java.nio.charset.StandardCharsets; import java.util.List; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -30,7 +31,7 @@ public class RuleFileMigrator { //replaceRules(ruleFileBluePrint, combinedBluePrint); replaceRuleIdentifiers(combinedBluePrint, ruleFileBluePrint); - String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); + String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java similarity index 76% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java index fc8e9026..3889fad1 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.FileOutputStream; import java.io.IOException; @@ -9,16 +9,15 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -35,7 +34,6 @@ public class RuleIdentifierMigrator { @SneakyThrows public void migrateAllRuleIdentifiers() { - List parsedRecords = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); RuleFileBluePrint bluePrint = getBluePrint(ApplicationType.RM); // migrateIdentifier(RuleIdentifier.fromString("PII.10.0"), RuleIdentifier.fromString("CBI.20.0"), bluePrint, parsedRecords); @@ -46,20 +44,17 @@ public class RuleIdentifierMigrator { bluePrint = migrateMatchedRuleForAllRules(bluePrint); - String ruleString = RuleFileFactory.buildRuleString(bluePrint); + String ruleString = RuleFileFactory.buildRuleString(bluePrint, false, false); try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) { out.write(ruleString.getBytes(StandardCharsets.UTF_8)); } - String csvString = OldRulesParser.formatAsCsv(parsedRecords); - try (var out = new FileOutputStream("/tmp/old_rules_with_translations.csv")) { - out.write(csvString.getBytes(StandardCharsets.UTF_8)); - } } + //todo: introduce functions and declarations private static RuleFileBluePrint migrateMatchedRuleForAllRules(RuleFileBluePrint bluePrint) { - List migratedRules = bluePrint.ruleClasses() + List migratedRules = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) @@ -67,7 +62,12 @@ public class RuleIdentifierMigrator { .flatMap(Collection::stream) .map(RuleIdentifierMigrator::migrateMatchedRule) .toList(); - RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.imports(), bluePrint.globals(), bluePrint.queries(), new LinkedList<>()); + RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.getImports(), + bluePrint.getGlobals(), + bluePrint.getQueries(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>()); migratedRules.forEach(migratedBluePrint::addRule); return migratedBluePrint; } @@ -118,7 +118,7 @@ public class RuleIdentifierMigrator { } - public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List records) { + public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint) { BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()) .orElseThrow().findRuleUnitByInteger(oldIdentifier.unit()) @@ -132,9 +132,6 @@ public class RuleIdentifierMigrator { bluePrint.removeRule(oldIdentifier); bluePrint.addRule(newRule); - records.stream() - .filter(record -> record.translatesTo().contains(oldIdentifier)) - .forEach(record -> replaceOldIdentifier(oldIdentifier, newIdentifier, record.translatesTo())); } @@ -152,7 +149,7 @@ public class RuleIdentifierMigrator { } - public record Context(RuleFileBluePrint bluePrint, List records) { + public record Context(RuleFileBluePrint bluePrint) { } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java similarity index 73% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java index 8fdac0d7..53d1122c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import lombok.Getter; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java new file mode 100644 index 00000000..4e3acb5c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicDeclaration { + + @EqualsAndHashCode.Include + String declarationFullTypeName; + String body; + int line; + + + public static BasicDeclaration fromDeclarationDescription(AbstractClassTypeDeclarationDescr declaration, String rulesString) { + + String declarationFullTypeName = declaration.getFullTypeName(); + String body = rulesString.substring(declaration.getStartCharacter(), declaration.getEndCharacter()); + int line = declaration.getLine(); + + return new BasicDeclaration(declarationFullTypeName, body, line); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java new file mode 100644 index 00000000..1793d589 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java @@ -0,0 +1,36 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.FunctionDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicFunction { + + @EqualsAndHashCode.Include + String functionName; + String returnType; + String body; + int line; + + + public static BasicFunction fromFunctionDescr(FunctionDescr function, String rulesString) { + + String functionName = function.getName(); + String returnType = function.getReturnType(); + String body = rulesString.substring(function.getStartCharacter(), function.getEndCharacter()); + int line = function.getLine(); + + return new BasicFunction(functionName, returnType, body, line); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java similarity index 80% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java index 9d77bbb6..cc5c0fe6 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java @@ -1,8 +1,8 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; public record BasicRule(RuleIdentifier identifier, String name, String code) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java index a6cdb0ab..bb0e2d79 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java index 40468cc0..e70acc0f 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.LinkedList; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java similarity index 85% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 2b2ae91f..28b2663c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Collection; import java.util.Collections; @@ -10,7 +10,29 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -public record RuleFileBluePrint(String imports, String globals, String queries, List ruleClasses) { +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RuleFileBluePrint { + + String imports; + String globals; + String queries; + List ruleClasses; + List declarations; + List functions; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -24,7 +46,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, ruleClass.ruleUnits().remove(ruleUnit); } if (ruleClass.ruleUnits().isEmpty()) { - ruleClasses().remove(ruleClass); + this.ruleClasses.remove(ruleClass); } })); return wasRemoved.get(); @@ -41,7 +63,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRules() { - return ruleClasses().stream() + return this.ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -101,7 +123,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRuleIdentifiers() { - return ruleClasses().stream() + return ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -122,10 +144,16 @@ public record RuleFileBluePrint(String imports, String globals, String queries, } + public void setImports(String newImports) { + + this.imports = newImports; + } + + public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>()); - ruleClasses().stream() + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) .flatMap(ruleUnit -> ruleUnit.rules() diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java similarity index 97% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java index 569412e4..3fce64a7 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Arrays; import java.util.Objects; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java similarity index 95% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java index ffddc766..fcacf44d 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Map; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java similarity index 54% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java index ec23ad27..a2b84ba3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java similarity index 96% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java index 3d3ccac6..8efecd93 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.utils; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.utils; import java.io.File; import java.io.FileInputStream; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java new file mode 100644 index 00000000..32aaf4fc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -0,0 +1,169 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class RuleBuilderService { + + //todo: make this configurable + private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), + new RuleType("MAN"), + new RuleType("X"), + new RuleType("DICT"), + new RuleType("FA"), + new RuleType("LDS"))); + + + public RuleBuilderModel getRuleBuilderModel() { + + RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + + ruleBuilderModel.setWhenClauses(Collections.emptyList()); + ruleBuilderModel.setThenConditions(Collections.emptyList()); + + return ruleBuilderModel; + } + + + public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + log.info("Starting to remove system rules from ruleFile"); + removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); + log.info("Finished removing system rules for ruleFile"); + + return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); + } + + + public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { + + log.info("starting to merge user rules update with system rules"); + RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + mergedRuleFileBlueprint.getRuleClasses() + .forEach(ruleClass -> { + if (systemRules.stream() + .map(RuleType::name) + .toList().contains(ruleClass.ruleType().name())) { + throw new RuntimeException("No system rule updates allowed in user rule update."); + } + }); + removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getRuleClasses() + .stream() + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .flatMap(ruleUnit -> ruleUnit.rules() + .stream())) + .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); + mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() + .stream(), + mergedRuleFileBlueprint.getFunctions() + .stream()) + .toList()); + mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() + .stream(), + mergedRuleFileBlueprint.getDeclarations() + .stream()) + .toList()); + log.info("finished merging user rules update with system rules"); + RuleMergingResult mergingResult = RuleMergingResult.builder() + .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) + .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) + .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .build(); + + return mergingResult; + } + + + private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { + + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + removeAllRulesExceptSystemRules(ruleFileBluePrint); + ruleFileBluePrint.getRuleClasses() + .forEach(ruleClass -> { + if (!systemRuleNames.contains(ruleClass.ruleType().name())) { + throw new RuntimeException("there was an error removing all rules except system rules"); + } + }); + } + + + private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + + List rules = new ArrayList(); + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + + ruleFileBluePrint.getRuleClasses() + .stream() + .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) + .forEach(rule -> rules.add(rule)); + rules.forEach(ruleIdentifier -> { + ruleFileBluePrint.removeRule(ruleIdentifier); + }); + } + + + private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { + + removeSystemRules(ruleFileBluePrint); + for (RuleType systemRule : systemRules) { + List remainingSystemRules = new ArrayList(); + ruleFileBluePrint.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(remainingSystemRules::add)); + if (!remainingSystemRules.isEmpty()) { + throw new RuntimeException("There was an error removing the system rules from the file"); + } + } + } + + + private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { + + for (RuleType systemRule : systemRules) { + List rules = new ArrayList(); + ruleFileBluePrintExisting.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(rules::add)); + rules.forEach(ruleUnit -> { + ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); + }); + ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index 9b983752..e27361be 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -69,51 +69,64 @@ public class DroolsValidationService { private DroolsValidation buildCustomDroolsValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation(); + DroolsValidation customValidation = ruleCompilationResult.getDroolsValidation(); - addSyntaxDeprecatedWarnings(ruleFileBluePrint, customValidation); + addSyntaxDeprecatedWarnings(ruleCompilationResult, customValidation); - addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxErrorMessages(ruleFileType, ruleCompilationResult, customValidation); if (redactionServiceSettings.isRuleExecutionSecured()) { - addBlacklistErrorMessages(ruleFileBluePrint, customValidation); + addBlacklistErrorMessages(ruleCompilationResult, customValidation); } return customValidation; } - private void addSyntaxDeprecatedWarnings(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxDeprecatedWarnings(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { // find deprecated elements in the ruleFileBluePrint - DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); + DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleCompilationResult); if (warningMessageForImports != null) { customValidation.getDeprecatedWarnings().add(warningMessageForImports); } + customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleCompilationResult)); + } + + + private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleCompilationResult ruleCompilationResult) { + + return DroolsSyntaxDeprecatedWarnings.builder() + .message("global ComponentMappingService " + + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL + + "\n is missing from the rules, consider adding it, as it will be required in future versions!") + .line(ruleCompilationResult.getGlobalsLine()) + .column(0) + .build(); customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } - private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { + private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleCompilationResult ruleCompilationResult) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { - String imports = ruleFileBluePrint.getImports(); + String imports = ruleCompilationResult.getImports(); SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation(); List matches = classesSearchImplementation.getMatches(imports); if (!matches.isEmpty()) { String sb = "Following imports are deprecated: \n" + matches.stream() .map(m -> imports.substring(m.startIndex(), m.endIndex())) .collect(Collectors.joining("\n")); - return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build(); + return DroolsSyntaxDeprecatedWarnings.builder().line(ruleCompilationResult.getImportLine()).column(0).message(sb).build(); } } return null; } - private List getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { + private List getWarningsForDeprecatedRules(RuleCompilationResult ruleCompilationResult) { List warningMessages = new ArrayList<>(); @@ -122,7 +135,7 @@ public class DroolsValidationService { SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation(); Map deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap(); - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { List matches = methodsSearchImplementation.getMatches(basicRule.getCode()); @@ -146,32 +159,32 @@ public class DroolsValidationService { } - private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { - RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { - case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); - case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + RuleCompilationResult baseRuleCompilationResult = switch (ruleFileType) { + case ENTITY -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); + case COMPONENT -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseComponentRuleFileString()); }; - if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) { + if (!importsAreValid(baseRuleCompilationResult, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getImportLine()) + .line(ruleCompilationResult.getImportLine()) .column(0) - .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports())) + .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleCompilationResult.getImports())) .build()); } - if (!ruleFileBluePrint.getGlobals().contains(baseRuleFileBluePrint.getGlobals())) { + if (!ruleCompilationResult.getGlobals().contains(baseRuleCompilationResult.getGlobals())) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) - .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals())) + .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleCompilationResult.getGlobals())) .build()); } - baseRuleFileBluePrint.getQueries() + baseRuleCompilationResult.getQueries() .forEach(basicQuery -> { - if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { + if (!validateQueryIsPresent(basicQuery, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(basicQuery.getLine()) @@ -182,7 +195,7 @@ public class DroolsValidationService { }); if (ruleFileType.equals(RuleFileType.ENTITY)) { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; - if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { + if (!validateAgendaGroupIsPresent(ruleCompilationResult, requiredAgendaGroup)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(0) @@ -194,18 +207,18 @@ public class DroolsValidationService { } - private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) { + private boolean validateAgendaGroupIsPresent(RuleCompilationResult ruleCompilationResult, String agendaGroupName) { - return ruleFileBluePrint.streamAllRules() + return ruleCompilationResult.streamAllRules() .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName)); } - private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) { + private boolean importsAreValid(RuleCompilationResult baseRuleCompilationResult, RuleCompilationResult ruleCompilationResult) { // imports may shrink, but not add anything new! - Set baseImports = baseRuleFileBluePrint.getImportSplitByKeyword(); - Set imports = ruleFileBluePrint.getImportSplitByKeyword(); + Set baseImports = baseRuleCompilationResult.getImportSplitByKeyword(); + Set imports = ruleCompilationResult.getImportSplitByKeyword(); Set additionalImports = Sets.difference(imports, baseImports); @@ -214,15 +227,15 @@ public class DroolsValidationService { } - private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) { + private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleCompilationResult ruleCompilationResult) { - return ruleFileBluePrint.getQueries() + return ruleCompilationResult.getQueries() .stream() .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode())); } - private void addBlacklistErrorMessages(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addBlacklistErrorMessages(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { List blacklistErrorMessages = new ArrayList<>(); @@ -234,14 +247,14 @@ public class DroolsValidationService { // check also the imports DroolsBlacklistErrorMessage blacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, - ruleFileBluePrint.getImports(), - ruleFileBluePrint.getImportLine()); + ruleCompilationResult.getImports(), + ruleCompilationResult.getImportLine()); if (blacklistErrorMessage != null) { blacklistErrorMessages.add(blacklistErrorMessage); } // check the rules - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { DroolsBlacklistErrorMessage ruleBlacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java similarity index 53% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index cd6a953e..616cabde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -22,21 +22,48 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.RuleType; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @UtilityClass -public class RuleFileParser { +public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { + + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); + List allRules = new LinkedList<>(); + List allQueries = new LinkedList<>(); + PackageDescr packageDescr = new PackageDescr(); + String imports = ""; + String globals = ""; + List ruleClasses = buildRuleClasses(allRules); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + @SneakyThrows - public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); @@ -52,12 +79,15 @@ public class RuleFileParser { } } - String imports = ruleString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + String imports = ""; + if (!removedImports) { + imports = ruleString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -65,21 +95,81 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + + @SneakyThrows + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { + + return buildRuleCompilationResultFromRuleString(ruleString,false); + } + + + @SneakyThrows + public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { + + StringBuilder ruleStringBuilder = new StringBuilder(); + + // Append imports + ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); + + // Append globals + if (!bluePrint.getGlobals().isEmpty()) { + ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); + } + + // Append queries + for (BasicQuery query : bluePrint.getQueries()) { + ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); + } + + // Append rules + for (RuleClass ruleClass : bluePrint.getRuleClasses()) { + ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); + } + + // Return the final rule string + return ruleStringBuilder.toString().trim(); + } + + + private String buildQueryString(BasicQuery query) { + + return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; + } + + + private String buildRuleString(RuleClass ruleClass) { + + StringBuilder ruleBuilder = new StringBuilder(); + + // Use RuleType to distinguish between different rule types, if needed + ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); + + for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { + for (BasicRule rule : ruleUnit.rules()) { + // Assuming BasicRule has a method to retrieve the condition as a string + ruleBuilder.append(rule.getCode()).append("\n"); + } + } + ruleBuilder.append("end"); + + return ruleBuilder.toString(); } @@ -126,6 +216,7 @@ public class RuleFileParser { private List buildRuleClasses(List allRules) { + //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv diff --git a/redaction-service-v1/rules-management/src/main/resources/order_template.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/order_template.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java new file mode 100644 index 00000000..1996af83 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -0,0 +1,149 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RuleBuilderTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); + private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); + private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + + @Autowired + RuleBuilderService ruleBuilderService; + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import(LayoutParsingServiceProcessorConfiguration.class) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RuleBuilderTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + public void removeSystemRulesTest() { + + String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); + String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + + Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + + Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + } + + + @Test + public void mergeUserRulesUpdateTest() { + + RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); + String mergedRules = mergingResult.getMergedRules(); + RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); + ruleFileBluePrintUserRulesUpdate.getRuleClasses() + .forEach(ruleClass -> { + Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); + }); + + } + + + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getRuleClasses() + .stream() + .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) + .collect(Collectors.toList()).size() == 0; + } + + + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getImports().isEmpty(); + } + + + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getQueries().isEmpty(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java index cac2af47..2d95017d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java @@ -11,10 +11,10 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.core.io.ClassPathResource; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index 4db674a4..0bd9ecec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -22,10 +22,10 @@ import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; import lombok.SneakyThrows; @@ -232,11 +232,11 @@ class DroolsValidationServiceTest { if (droolsValidation.isCompiled()) { continue; } - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(rulesString); + RuleCompilationResult baseRuleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); - rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); - rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + rulesString = rulesString.replace(ruleCompilationResult.getImports(), baseRuleCompilationResult.getImports()); + rulesString = rulesString.replace(ruleCompilationResult.getGlobals(), baseRuleCompilationResult.getGlobals()); try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); @@ -457,9 +457,9 @@ class DroolsValidationServiceTest { end """; - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled()); + assertFalse(ruleCompilationResult.getDroolsValidation().isCompiled()); } } \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java similarity index 91% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java index f8d25b6a..fa77a92b 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,10 +27,10 @@ public class DroolsCompilationTest { @Test public void testValidateRuleSyntax() throws IOException { - URL rmURL = Resources.getResource("all_redact_manager_rules.drl"); + URL rmURL = Resources.getResource("rulesmanagement/all_redact_manager_rules.drl"); String rmRule = Resources.toString(rmURL, StandardCharsets.UTF_8); - URL dmURL = Resources.getResource("all_rules_documine.drl"); + URL dmURL = Resources.getResource("rulesmanagement/all_rules_documine.drl"); String dmRule = Resources.toString(dmURL, StandardCharsets.UTF_8); assertTrue(validateRuleSyntax(rmRule).isValid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java new file mode 100644 index 00000000..577f996a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; + +public class RuleFileBluePrintMergingTest { + + @Test + public void testBothRuleFilesCanBeMerged() { + + RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); + assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); + } + +} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java similarity index 93% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java index c3629d9c..be926def 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import java.io.File; import java.nio.file.Files; @@ -8,7 +8,7 @@ import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.migration.RuleFileMigrator; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java similarity index 94% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java index b6b85cef..ce84ebbf 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -19,10 +19,10 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl new file mode 100644 index 00000000..16ea73d2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl @@ -0,0 +1,50 @@ +package drools +//------------------------------------ Table extraction rules ------------------------------------ + +// Rule unit: TAB.0 +rule "TAB.0.0: Changed Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Changed Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +// Rule unit: TAB.6 +rule "TAB.6.1: Changed Targeted cell extraction (Experimental Stop date)" + when + $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2")) + $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList() + TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList() + $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList() + then + entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date")); + end + diff --git a/redaction-service-v1/rules-management/build.gradle.kts b/redaction-service-v1/rules-management/build.gradle.kts index 2aaea40c..b005ed7e 100644 --- a/redaction-service-v1/rules-management/build.gradle.kts +++ b/redaction-service-v1/rules-management/build.gradle.kts @@ -27,10 +27,10 @@ sourceSets { } dependencies { - implementation(project(":redaction-service-server-v1")) + implementation(project(mapOf("path" to ":redaction-service-server-v1"))) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") - + implementation(project(":redaction-service-server-v1")) implementation("com.github.javaparser:javaparser-core:3.25.3") implementation("org.drools:drools-drl-parser:8.41.0.Final") implementation("org.apache.commons:commons-csv:1.10.0") diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java index 7267bd20..6ab58cf3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java @@ -17,12 +17,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; @@ -72,7 +71,6 @@ public class Main { Set identifiers = switch (type) { case "list", "l" -> RuleIdentifier.fromListOfIdentifiersString(input); case "file", "f" -> RuleFileParser.parseRuleIdentifiersFromFile(input); - case "old", "o" -> OldRulesParser.translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(input), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid", cmd.getOptionValue("t"))); }; ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType); @@ -102,7 +100,6 @@ public class Main { String type = cmd.hasOption("t") ? cmd.getOptionValue("t") : "f"; Map> identifiersPerFile = switch (type) { case "file", "f" -> parseIdentifiersFromFiles(Path.of(cmd.getOptionValue("input"))); - case "old", "o" -> translateIdentifiersFromOldFiles(Path.of(cmd.getOptionValue("input")), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid in combination with the \"recursive\" flag", cmd.getOptionValue("t"))); }; identifiersPerFile.keySet() @@ -137,13 +134,6 @@ public class Main { } - @SneakyThrows - private static Map> translateIdentifiersFromOldFiles(Path inputDirectory, ApplicationType applicationType) { - - return RuleFileIO.streamAllRuleFilesInDirectory(inputDirectory) - .collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, applicationType))); - } - @SneakyThrows private static Map> parseIdentifiersFromFiles(Path inputDirectory) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java deleted file mode 100644 index af844a6d..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -public class RuleManagementResources { - - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_redact_manager_rules.drl"); - } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl"); - } - - - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers.txt"); - } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers_dm.txt"); - } - } - - - public static InputStream getTemplateInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("order_template.txt"); - } - - - public static InputStream getOldRulesCsvInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("old_rules_with_translations.csv"); - } - - - @SneakyThrows - public static String createTempOldRulesCsv(String formattedAsCsv) { - - File csvFile = File.createTempFile("old_rules_with_translations-", ".csv"); - try (var out = new FileOutputStream(csvFile)) { - out.write(formattedAsCsv.getBytes(StandardCharsets.UTF_8)); - } - return csvFile.toString(); - } - -} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java deleted file mode 100644 index ae0f3134..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java +++ /dev/null @@ -1,203 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; -import org.drools.drl.ast.descr.PackageDescr; -import org.drools.drl.ast.descr.RuleDescr; -import org.drools.drl.parser.DrlParser; -import org.kie.internal.builder.conf.LanguageLevelOption; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.OldRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; - -@UtilityClass -public class OldRulesParser { - - static List HEADERS = List.of("id", "old rule names", "old rule code", "translates to"); - - - @SneakyThrows - public void createSetOfTranslatedRuleIdentifiersForEachFileInTheCsv() { - - List allHeaders = List.of(); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - - } - - - public Set translateOldRulesFileToNewIdentifiers(File oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesFileToNewIdentifiers(oldRulesFile.toString(), applicationType); - } - - - public Set translateOldRulesFileToNewIdentifiers(String oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(oldRulesFile), applicationType); - } - - - public Set translateEscapedOldRulesStringToNewIdentifiers(String escapedOldRulesString, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.unescapeAndUnWrap(escapedOldRulesString), applicationType); - } - - - public Set translateOldRulesStringToNewIdentifiers(String oldRulesString, ApplicationType applicationType) { - - List oldRules = parseOldRules(oldRulesString); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - Map> translationPairs = new HashMap<>(); - for (OldRule oldRule : oldRules) { - List translatedIdentifiers = records.stream() - .filter(oldRulesCsvRecord -> oldRulesCsvRecord.code().equals(oldRule.code())) - .map(OldRulesCsvRecord::translatesTo) - .findAny() - .orElse(Collections.emptyList()); - translationPairs.put(oldRule, translatedIdentifiers); - } - boolean allTranslated = true; - for (OldRule oldRule : translationPairs.keySet()) { - if (translationPairs.get(oldRule).isEmpty()) { - allTranslated = false; - records.add(new OldRulesCsvRecord(records.size(), oldRule.code(), oldRule.name(), Collections.emptyList())); - System.out.printf( - "Rule %s has not been translated yet! %nIt has been added to the bottom of the old_rules_with_translations.csv file. To continue, translate the rule and enter its identifier in the csv, then try again!%n", - oldRule.name()); - } - } - if (!allTranslated) { - String formattedAsCsv = formatAsCsv(records); - String fileLocation = RuleManagementResources.createTempOldRulesCsv(formattedAsCsv); - System.out.printf("CSV File with updated values is located at %s%n", fileLocation); - throw new IllegalArgumentException("Non translated Rule found!"); - } - RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); - - translationPairs.forEach((key, value) -> System.out.printf("Rule %s will be translated to %s \n", - key.name(), - String.format("%s: %s", - value, - value.stream() - .map(bluePrint::findRuleByIdentifier) - .flatMap(Collection::stream) - .map(BasicRule::name) - .toList()))); - - return Stream.concat(// - Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),// - translationPairs.values() - .stream() - .flatMap(Collection::stream)) - .map(ruleIdentifier -> new RuleIdentifier(ruleIdentifier.type(), ruleIdentifier.unit(), null)) - .collect(Collectors.toSet()); - } - - - @SneakyThrows - public String formatAsCsv(List records) { - - StringWriter sw = new StringWriter(); - CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS.toArray(String[]::new)).build(); - try (CSVPrinter printer = new CSVPrinter(sw, csvFormat)) { - for (OldRulesCsvRecord record : records) { - printer.printRecord(Stream.of(record.id, RuleFileIO.escapeAndWrap(record.names), RuleFileIO.escapeAndWrap(record.code), record.translatesTo)); - } - } - return sw.toString(); - } - - - @SneakyThrows - private List parseOldRules(String oldRuleString) { - - DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); - PackageDescr packageDescr = parser.parse(false, oldRuleString); - List oldRules = new LinkedList<>(); - for (RuleDescr rule : packageDescr.getRules()) { - if (!rule.isRule()) { - continue; - } - oldRules.add(OldRule.fromRuleDescr(rule, oldRuleString)); - } - return oldRules; - } - - - @SneakyThrows - public List getOldRulesCsvRecords(InputStream allRulesCsvInputStream) { - - List parsedRecords = new LinkedList<>(); - try (Reader reader = new InputStreamReader(allRulesCsvInputStream)) { - Iterable records = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord().parse(reader); - for (CSVRecord record : records) { - String[] values = record.values(); - parsedRecords.add(new OldRulesCsvRecord(Long.parseLong(values[0]), - RuleFileIO.unescapeAndUnWrap(values[2]), - RuleFileIO.unescapeAndUnWrap(values[1]), - parseRuleIdentifiers(values[3]))); - } - } - return parsedRecords; - } - - - private List parseRuleIdentifiers(String value) { - - String cleanedValue = value; - if (cleanedValue.startsWith("[")) { - cleanedValue = cleanedValue.substring(1); - } - if (cleanedValue.endsWith("]")) { - cleanedValue = cleanedValue.substring(0, cleanedValue.length() - 1); - } - if (cleanedValue.isEmpty() || cleanedValue.isBlank()) { - return Collections.emptyList(); - } - List ruleIdentifiers = new LinkedList<>(); - for (String identifier : cleanedValue.split(", ")) { - ruleIdentifiers.add(RuleIdentifier.fromString(identifier)); - } - return ruleIdentifiers; - } - - - private List parseBooleanList(String[] values) { - - return Arrays.stream(values) - .map(Boolean::parseBoolean) - .toList(); - } - - - public record OldRulesCsvRecord(long id, String code, String names, List translatesTo) { - - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java deleted file mode 100644 index 559101ac..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; - -public class RuleFileBluePrintMergingTest { - - @Test - public void testBothRuleFilesCanBeMerged() { - - RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); - assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java deleted file mode 100644 index b78ab534..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -class OldRulesParserTest { - - @Test - @SneakyThrows - @Disabled - public void translateOldRulesToNewIdentifiersTest() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("efsa_regulation_rules.txt")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateEscapedOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @SneakyThrows - @Disabled - public void translateMoreOldRules() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("basf/demo/table_demo.drl")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @Disabled - public void printTranslationsTest() { - - List records = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")) - .forEach(type -> { - List rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type) - .orElseThrow().ruleUnits(); - rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records)); - }); - } - - - private void printOldRulesThatTranslatesToNewRule(RuleUnit ruleUnit, List records) { - - if (ruleUnit.rules().isEmpty()) { - System.out.println("Rule unit empty, skipping!\n"); - return; - } - RuleIdentifier identifier = ruleUnit.rules() - .get(0).identifier(); - RuleIdentifier unitIdentifier = new RuleIdentifier(identifier.type(), identifier.unit(), null); - String oldNames = records.stream() - .filter(r -> r.translatesTo() - .stream() - .anyMatch(i -> i.matches(unitIdentifier))) - .map(OldRulesParser.OldRulesCsvRecord::names) - .map(OldRulesParserTest::removeIdFromName) - .distinct() - .collect(Collectors.joining(", ")); - System.out.println(unitIdentifier.toRuleUnitString() + " " + ruleUnit.rules() - .stream() - .map(BasicRule::name) - .collect(Collectors.joining(", "))); - System.out.println("translate from"); - System.out.println(oldNames); - System.out.println(); - System.out.println(); - } - - - private static String removeIdFromName(String name) { - - String[] values = name.split(":"); - return String.join(":", Arrays.copyOfRange(values, 1, values.length)); - } - - - @Test - @Disabled - @SneakyThrows - public void findAllCustomerRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("pilot/EFSA_sanitisation_GFL_v1").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.RM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.RM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void findAllRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("business-logic/Syngenta_RSS").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.DM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.DM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void translateDossierTemplatesV2() { - -// String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/RED/dossier-templates-v2/"; -// Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), Files.walk(Path.of(dossierTemplatesRepo + "docu")), Files.walk(Path.of(dossierTemplatesRepo + "qa"))) - String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/"; - Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), - Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) - // - ) - .flatMap(Function.identity())// - .filter(path -> path.getFileName().toString().equals("rules.drl"))// - .map(Path::toFile)// - .forEach(this::translateOldRuleFile); - - } - - - @SneakyThrows - private void translateOldRuleFile(File oldRulesFile) { - - Set identifiers = OldRulesParser.translateOldRulesFileToNewIdentifiers(oldRulesFile, ApplicationType.RM); - String newRulesString = RuleFileFactory.createFileFromIdentifiers(identifiers, ApplicationType.RM); - String result = RuleFileIO.escapeAndWrap(newRulesString); - - try (var out = new FileOutputStream(oldRulesFile)) { - out.write(result.getBytes(StandardCharsets.UTF_8)); - } - } - -} \ No newline at end of file -- 2.47.2 From e67d453d082de29919c88b96ba93315c7050ee44 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 09:44:51 +0100 Subject: [PATCH 033/108] RED-9472: seperation of system rules code review changes --- .../controller/RuleBuilderController.java | 9 +- .../model/drools/RuleCompilationResult.java | 46 +---- .../RuleManagementResources.java | 19 +- .../factory/RuleFileParser.java | 15 +- .../models/RuleFileBluePrint.java | 28 +++ .../v1/server/service/RuleBuilderService.java | 91 ++------- .../drools/RuleCompilationResultParser.java | 83 +------- .../redaction/v1/server/RuleBuilderTest.java | 183 ++++++++---------- 8 files changed, 147 insertions(+), 327 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 8f3a51dd..aa215fc7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -45,7 +45,7 @@ public class RuleBuilderController implements RuleBuilderResource { RulesResponse rulesResponse = new RulesResponse(); String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); if (filteredRules.isEmpty()) { - throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + throw new AssertionError("There was an error when cleaning the rulefile of sytem rules"); } rulesResponse.setRules(filteredRules); return rulesResponse; @@ -53,13 +53,13 @@ public class RuleBuilderController implements RuleBuilderResource { @Override - public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); DroolsValidationResponse droolsValidationResponse; RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); if (mergingResult.getMergedRules().isEmpty()) { - throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); @@ -90,7 +90,8 @@ public class RuleBuilderController implements RuleBuilderResource { .toList()) .build(); if (!droolsValidation.isCompiled()) { - return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.UNPROCESSABLE_ENTITY); } else { // rulesUploadResponse.setRules(mergingResult.getMergedRules()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index e96aa697..3506a31f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -49,60 +49,16 @@ public final class RuleCompilationResult { public void dropRulesByIdentifier(RuleType ruleType) { - log.info("removing rule with identifier {}", ruleType.name()); - List rulesToBeRemoved = ruleClasses.stream() .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); ruleClasses.removeAll(rulesToBeRemoved); } - public void dropAllRulesExceptSystemRules(List systemRules) { - - log.info("removing all rules except the system rules"); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) - .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); - ruleClasses.removeAll(rulesToBeRemoved); - } - - - public void dropImports() { - - this.imports = ""; - } - - - public void dropQueries() { - - this.queries.clear(); - } - - - public int countRuleOccurences(RuleType ruleType) { - - log.info("counting occurences of files {}", ruleType.name()); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) - .toList(); - return rulesToBeRemoved.size(); - } - - - private boolean filterOutRule(RuleType ruleType, List filteredRules) { - - return !filteredRules.contains(ruleType.name()); - } - - public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replace("\n", "").split("import")) + return Arrays.stream(imports.replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java index 331b1e25..e423dde8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -1,34 +1,37 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement; +import java.io.IOException; import java.io.InputStream; +import org.drools.io.ClassPathResource; + import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; @SuppressWarnings("PMD") public class RuleManagementResources { - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + public static InputStream getAllRulesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + return new ClassPathResource("drools/all_redact_manager_rules.drl").getInputStream(); } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + return new ClassPathResource("drools/all_rules_documine.drl").getInputStream(); } - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers.txt").getInputStream(); } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers_dm.txt").getInputStream(); } } - public static InputStream getTemplateInputStream() { + public static InputStream getTemplateInputStream() throws IOException { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + return new ClassPathResource("rulesmanagement/order_template.txt").getInputStream(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index db0ece77..6354f93c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -4,7 +4,6 @@ import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -121,16 +120,12 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - if (rulesPerUnit.keySet() != null) { - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); - } else { - return new ArrayList<>(); - } + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 28b2663c..ca185b63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; @@ -53,6 +54,33 @@ public class RuleFileBluePrint { } + public boolean removeRuleClassByRuleIdentifier(RuleIdentifier ruleIdentifier) { + + AtomicBoolean wasRemoved = new AtomicBoolean(false); + + findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> { + List unitsToRemove = new ArrayList<>(); + + ruleClass.ruleUnits() + .forEach(ruleUnit -> { + boolean removed = ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier)); + if (removed) { + wasRemoved.set(true); + if (ruleUnit.rules().isEmpty()) { + unitsToRemove.add(ruleUnit); + } + } + }); + + ruleClass.ruleUnits().removeAll(unitsToRemove); + if (ruleClass.ruleUnits().isEmpty()) { + this.ruleClasses.remove(ruleClass); + } + }); + return wasRemoved.get(); + } + + public Optional findRuleClassByType(RuleType ruleType) { return ruleClasses.stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 32aaf4fc..d20e8337 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -3,9 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -16,8 +15,6 @@ import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFil import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.extern.slf4j.Slf4j; @@ -25,13 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - //todo: make this configurable - private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), - new RuleType("MAN"), - new RuleType("X"), - new RuleType("DICT"), - new RuleType("FA"), - new RuleType("LDS"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); public RuleBuilderModel getRuleBuilderModel() { @@ -48,28 +44,24 @@ public class RuleBuilderService { public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - log.info("Starting to remove system rules from ruleFile"); - removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); - log.info("Finished removing system rules for ruleFile"); - + removeSystemRules(ruleFileBluePrint); return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); } public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { - log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() - .map(RuleType::name) - .toList().contains(ruleClass.ruleType().name())) { - throw new RuntimeException("No system rule updates allowed in user rule update."); + .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { + log.warn("System rule in user rule update."); + //throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); ruleFileBluePrintExisting.getRuleClasses() .stream() .flatMap(ruleClass -> ruleClass.ruleUnits() @@ -89,7 +81,6 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - log.info("finished merging user rules update with system rules"); RuleMergingResult mergingResult = RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) @@ -100,69 +91,17 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { - - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); - removeAllRulesExceptSystemRules(ruleFileBluePrint); - ruleFileBluePrint.getRuleClasses() - .forEach(ruleClass -> { - if (!systemRuleNames.contains(ruleClass.ruleType().name())) { - throw new RuntimeException("there was an error removing all rules except system rules"); - } - }); - } - - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - List rules = new ArrayList(); - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); - ruleFileBluePrint.getRuleClasses() - .stream() - .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) - .forEach(rule -> rules.add(rule)); - rules.forEach(ruleIdentifier -> { - ruleFileBluePrint.removeRule(ruleIdentifier); - }); - } - - - private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { - - removeSystemRules(ruleFileBluePrint); - for (RuleType systemRule : systemRules) { - List remainingSystemRules = new ArrayList(); - ruleFileBluePrint.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(remainingSystemRules::add)); - if (!remainingSystemRules.isEmpty()) { - throw new RuntimeException("There was an error removing the system rules from the file"); - } - } } private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { - for (RuleType systemRule : systemRules) { - List rules = new ArrayList(); - ruleFileBluePrintExisting.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(rules::add)); - rules.forEach(ruleUnit -> { - ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); - }); - ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + for (RuleIdentifier systemRule : systemRules) { + ruleFileBluePrintExisting.removeRuleClassByRuleIdentifier(systemRule); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 616cabde..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -35,33 +35,6 @@ public class RuleCompilationResultParser { "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { - - DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); - List allRules = new LinkedList<>(); - List allQueries = new LinkedList<>(); - PackageDescr packageDescr = new PackageDescr(); - String imports = ""; - String globals = ""; - List ruleClasses = buildRuleClasses(allRules); - return new RuleCompilationResult(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); - } - - @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { @@ -116,60 +89,7 @@ public class RuleCompilationResultParser { @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { - return buildRuleCompilationResultFromRuleString(ruleString,false); - } - - - @SneakyThrows - public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { - - StringBuilder ruleStringBuilder = new StringBuilder(); - - // Append imports - ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); - - // Append globals - if (!bluePrint.getGlobals().isEmpty()) { - ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); - } - - // Append queries - for (BasicQuery query : bluePrint.getQueries()) { - ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); - } - - // Append rules - for (RuleClass ruleClass : bluePrint.getRuleClasses()) { - ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); - } - - // Return the final rule string - return ruleStringBuilder.toString().trim(); - } - - - private String buildQueryString(BasicQuery query) { - - return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; - } - - - private String buildRuleString(RuleClass ruleClass) { - - StringBuilder ruleBuilder = new StringBuilder(); - - // Use RuleType to distinguish between different rule types, if needed - ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); - - for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { - for (BasicRule rule : ruleUnit.rules()) { - // Assuming BasicRule has a method to retrieve the condition as a string - ruleBuilder.append(rule.getCode()).append("\n"); - } - } - ruleBuilder.append("end"); - - return ruleBuilder.toString(); + return buildRuleCompilationResultFromRuleString(ruleString, false); } @@ -216,7 +136,6 @@ public class RuleCompilationResultParser { private List buildRuleClasses(List allRules) { - //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 1996af83..181e108a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,149 +1,128 @@ package com.iqser.red.service.redaction.v1.server; +import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; import static org.mockito.Mockito.when; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; -import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; -import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class RuleBuilderTest extends AbstractRedactionIntegrationTest { +import org.junit.jupiter.api.Assertions; +import lombok.SneakyThrows; - private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); - private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); - private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); - - @Autowired - RuleBuilderService ruleBuilderService; - - @Configuration - @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) - @Import(LayoutParsingServiceProcessorConfiguration.class) - @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) - public static class RuleBuilderTestConfiguration { - - @Bean - @Primary - public StorageService inmemoryStorage() { - - return new FileSystemBackedStorageService(ObjectMapperFactory.create()); - } - - } +class RuleBuilderTest { + private RuleBuilderService ruleBuilderService; + private final List systemRules = new ArrayList<>(Arrays.asList( + RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*") + )); @BeforeEach - public void stubClients() { - + void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); - - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); - - loadDictionaryForTest(); - loadTypeForTest(); - loadNerForTest(); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); - - when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); - - mockDictionaryCalls(null); - - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } - @Test - public void removeSystemRulesTest() { + @SneakyThrows + void removeSystemRulesTest() throws IOException { + URL url = getClass().getClassLoader().getResource("drools"); + if (url == null) { + throw new RuntimeException("Drools directory not found in resources"); + } - String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); - String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); - RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + Path path = Paths.get(url.toURI()); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); - Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); - - Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } + private void testThatSystemRulesAreRemoved(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + true + ); - @Test - public void mergeUserRulesUpdateTest() { + RuleCompilationResult ruleCompilationResultWithImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); - RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); - String mergedRules = mergingResult.getMergedRules(); - RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); - ruleFileBluePrintUserRulesUpdate.getRuleClasses() - .forEach(ruleClass -> { - Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); - }); + Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports + ); + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + + Assertions.assertEquals( + RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + } private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getRuleClasses() - .stream() - .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) - .collect(Collectors.toList()).size() == 0; + .stream() + .map(RuleClass::ruleType) + .noneMatch(ruleType -> systemRules.stream() + .map(RuleIdentifier::type) + .anyMatch(type -> ruleType.name().equals(type.name()))); } - private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getImports().isEmpty(); } - private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getQueries().isEmpty(); } - -} +} \ No newline at end of file -- 2.47.2 From ef19d0dc596915ba72b134245afbac9166f5c1ea Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 034/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 5becb48bca9792d51fa17b51e6453570e4fc1b89 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 035/108] RED-9472: seperation of system rules pmd fix --- .../v1/server/service/RuleBuilderService.java | 20 +++++++++---------- .../drools/RuleCompilationResultParser.java | 8 ++------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index d20e8337..2c9c9f7e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -58,7 +58,7 @@ public class RuleBuilderService { if (systemRules.stream() .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { log.warn("System rule in user rule update."); - //throw new RuntimeException("No system rule updates allowed in user rule update."); + throw new RuntimeException("No system rule updates allowed in user rule update."); } }); removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); @@ -81,19 +81,17 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - RuleMergingResult mergingResult = RuleMergingResult.builder() + return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) .build(); - - return mergingResult; } private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 15a91bb3e67a1b5f3f9a5c826db77a16c6063ae3 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:26:32 +0100 Subject: [PATCH 036/108] RED-9472: seperation of system rules pmd fix --- .../redaction/v1/server/RuleBuilderTest.java | 83 ++++++++----------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 181e108a..31d61e1e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server; -import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URL; import java.nio.file.Path; @@ -13,8 +8,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; @@ -27,35 +24,32 @@ import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.knecon.fforesight.tenantcommons.TenantContext; -import org.junit.jupiter.api.Assertions; import lombok.SneakyThrows; class RuleBuilderTest { private RuleBuilderService ruleBuilderService; - private final List systemRules = new ArrayList<>(Arrays.asList( - RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*") - )); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); + @BeforeEach void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); } + @Test @SneakyThrows void removeSystemRulesTest() throws IOException { - URL url = getClass().getClassLoader().getResource("drools"); - if (url == null) { - throw new RuntimeException("Drools directory not found in resources"); - } + URL url = Thread.currentThread().getContextClassLoader().getResource("drools"); Path path = Paths.get(url.toURI()); RuleFileIO.streamAllRuleFilesInDirectory(path) .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); @@ -64,20 +58,14 @@ class RuleBuilderTest { .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } - private void testThatSystemRulesAreRemoved(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - true - ); - RuleCompilationResult ruleCompilationResultWithImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + private void testThatSystemRulesAreRemoved(Path path) { + + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), true); + + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); @@ -88,28 +76,22 @@ class RuleBuilderTest { Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - cleanedRulesWithImports - ); + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports); - Assertions.assertEquals( - RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(cleanedRuleFileMergedWithSystemRules.getMergedRules()); + + Assertions.assertEquals(RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), cleanedRuleFileMergedWithSystemRules.getMergedRules()); } + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getRuleClasses() .stream() .map(RuleClass::ruleType) @@ -118,11 +100,16 @@ class RuleBuilderTest { .anyMatch(type -> ruleType.name().equals(type.name()))); } + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getImports().isEmpty(); } + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getQueries().isEmpty(); } + } \ No newline at end of file -- 2.47.2 From eb91cd48bb03e7df4c18cea282a45fc332599f21 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 15 Nov 2024 09:45:56 +0100 Subject: [PATCH 037/108] RED-9472: seperation of system rules review changes --- .../models/RuleFileBluePrint.java | 2 +- .../v1/server/service/RuleBuilderService.java | 46 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index ca185b63..533492e1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -180,7 +180,7 @@ public class RuleFileBluePrint { public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), declarations, functions); ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 2c9c9f7e..c346429c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -1,10 +1,10 @@ package com.iqser.red.service.redaction.v1.server.service; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), - RuleIdentifier.fromName("MAN"), - RuleIdentifier.fromName("X"), - RuleIdentifier.fromName("DICT"), - RuleIdentifier.fromName("FA"), - RuleIdentifier.fromName("LDS"))); + private final HashSet systemRules = new HashSet<>(Set.of(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -52,7 +52,7 @@ public class RuleBuilderService { public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() @@ -61,25 +61,30 @@ public class RuleBuilderService { throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - ruleFileBluePrintExisting.getRuleClasses() - .stream() - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .flatMap(ruleUnit -> ruleUnit.rules() - .stream())) + ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getAllRules() .forEach(mergedRuleFileBlueprint::addRule); - mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); - mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setImports(uniqueImports.stream() + .collect(Collectors.joining(""))); + Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setGlobals(uniqueGlobals.stream() + .collect(Collectors.joining(""))); mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() .stream(), mergedRuleFileBlueprint.getFunctions() .stream()) + .distinct() .toList()); mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() .stream(), mergedRuleFileBlueprint.getDeclarations() .stream()) + .distinct() .toList()); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) @@ -89,10 +94,9 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + private RuleFileBluePrint removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + return ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } -- 2.47.2 From a767df39d37512fc7d51aac915cff7e5e97d325a Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 038/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 682422aba1e331343774052e14138b93d1b44a7f Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 039/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 22b529f90cf084a460acdbb6f1302df3aefbf20b Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 09:04:46 +0100 Subject: [PATCH 040/108] RED-9472: seperation of system rules added document dependency --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index f3d723ae..d2524cb6 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -38,7 +38,7 @@ configurations.all { } dependencies { - + implementation(project(":document")) implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") -- 2.47.2 From 06eec284abbc19991b84bcd4427cfd69727301a9 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 13:02:03 +0100 Subject: [PATCH 041/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index d2524cb6..7a308094 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -45,6 +45,7 @@ dependencies { { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } + implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}") implementation("com.iqser.red.commons:spring-commons:6.2.0") @@ -62,6 +63,7 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") implementation("org.ahocorasick:ahocorasick:0.6.3") + implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") -- 2.47.2 From d5d8eb2061584e9018c36e4401f57fba69a3333f Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Sep 2024 10:24:59 +0200 Subject: [PATCH 042/108] RED-9472: seperation of system rules added endpoints for seperating and readding system rules working on service RED-9472: seperation of system rules refactored endpoints. now removing system rules on download merging user rules with systemrules on upload RED-9472: seperation of system rules RED-9472: seperation of system rules RED-9472: seperation of system rules refactored two have a rulefileblueprint and a rulefileblueprint for validationresult changed the flow of merging and seperating system and user rules added functions and declarations to rule file parsing RED-9472: seperation of system rules RED-9472: seperation of system rules introduced tests working on fixing bug with merging rules RED-9472: seperation of system rules introduced tests RED-9472: seperation of system rules fixed circular depdency RED-9472: seperation of system rules updated persistence service version RED-9472: seperation of system rules refactored rules refactored pcackages RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed some sonarqube errors RED-9472: seperation of system rules fixed pmd error RED-9472: seperation of system rules added deploy stage to gitlab pipeline RED-9472: seperation of system rules RED-9472: seperation of system rules fixed a mistake RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../redaction-service-server-v1/build.gradle.kts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index dc942464..ccff795e 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.612.0-RED10072.1" +val persistenceServiceVersion = "2.585.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 7a308094..707d8690 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,8 +16,8 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.612.0-RED10072.1" -val llmServiceVersion = "1.20.0-RED10072.2" +val persistenceServiceVersion = "2.585.0-RED9472.1" +val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" -- 2.47.2 From af00a4f5caea80f022554aab4bae68ae3780caf0 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 043/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From c71668adff78b5dbe2bdcd431d4c65b79d18d765 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 044/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 802d663023d40db9dd96f3f745445332f9794cf3 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 045/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 0be1812c3a287a08ceb11f7c5394425750aaf9d3 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 046/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 8099676c2ccb9bbe720f3feb7c6e89546c555d72 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 13:02:03 +0100 Subject: [PATCH 047/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 707d8690..67c38388 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.585.0-RED9472.1" +val persistenceServiceVersion = "2.627.0-RED9472.2" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" -- 2.47.2 From 1642e14e3934833ada8439e3c67369be5b838f51 Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 08:52:32 +0100 Subject: [PATCH 048/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 67c38388..2714afe8 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.627.0-RED9472.2" +val persistenceServiceVersion = "2.628.0-RED9472.0" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" -- 2.47.2 From 791591137509c6f2ceec886bd4a40a4f5ff5601e Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 09:55:55 +0100 Subject: [PATCH 049/108] RED-9472: seperation of system rules added updated persistence service version --- redaction-service-v1/redaction-service-api-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index ccff795e..811fa166 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.585.0-RED9472.0" +val persistenceServiceVersion = "2.628.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") -- 2.47.2 From 2762d378d59072dec0b1e9aa1ef15fcde9b1986c Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 10:04:51 +0100 Subject: [PATCH 050/108] RED-9472: seperation of system rules added updated service version for fixing checkstyle errors --- .../redaction-service-server-v1/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 2714afe8..36f2fbb9 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -53,7 +53,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.50.0") - implementation("com.knecon.fforesight:tenant-commons:0.30.0") + implementation("com.knecon.fforesight:tenant-commons:0.31.0") implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } @@ -64,6 +64,8 @@ dependencies { implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") implementation("org.ahocorasick:ahocorasick:0.6.3") implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") + implementation("com.github.roklenarcic:aho-corasick:1.2") + implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") -- 2.47.2 From ea3b0b999938041ff983819aa1672ba820f3009d Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 10:18:10 +0100 Subject: [PATCH 051/108] RED-9472: seperation of system rules added updated service version for fixing checkstyle errors --- .../v1/server/service/drools/DroolsValidationService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index e27361be..48467c63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -105,7 +105,6 @@ public class DroolsValidationService { .line(ruleCompilationResult.getGlobalsLine()) .column(0) .build(); - customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } -- 2.47.2 From a011a93fd2fc44267e70b29d9ba037edd24bdf57 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Dec 2024 13:35:51 +0100 Subject: [PATCH 052/108] RED-9472: seperation of system rules --- .../service/redaction/v1/server/service/RuleBuilderService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index c346429c..54670ba4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -86,6 +86,7 @@ public class RuleBuilderService { .stream()) .distinct() .toList()); + log.info("merged rules: {}", RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) -- 2.47.2 From fb639b47576343e1b3bab1943c5264a1ebffaeff Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Dec 2024 15:00:01 +0100 Subject: [PATCH 053/108] RED-9472: seperation of system rules --- redaction-service-v1/redaction-service-api-v1/build.gradle.kts | 2 +- .../redaction-service-server-v1/build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 811fa166..97d4351d 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.628.0-RED9472.0" +val persistenceServiceVersion = "2.649.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 29b5aec8..477ca775 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.191.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.628.0-RED9472.0" +val persistenceServiceVersion = "2.649.0-RED9472.0" val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" -- 2.47.2 From b12457cd7b2541ce92beedd455624044b0b173ba Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Dec 2024 15:04:47 +0100 Subject: [PATCH 054/108] RED-9472: seperation of system rules --- .../files/management/services/DroolsValidationServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index 2e42db62..7fabeda2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -23,11 +23,11 @@ import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService; import lombok.SneakyThrows; -- 2.47.2 From a0b34894eed50043791e260ebf0dd6928e528831 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 11:52:36 +0100 Subject: [PATCH 055/108] RED-9472: seperation of system rules now adding user update rules to existing blueprint and returning this --- .../controller/RuleBuilderController.java | 2 +- .../v1/server/service/RuleBuilderService.java | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index aa215fc7..8c452b07 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -93,13 +93,13 @@ public class RuleBuilderController implements RuleBuilderResource { rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.UNPROCESSABLE_ENTITY); } else { - // rulesUploadResponse.setRules(mergingResult.getMergedRules()); rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); } } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } + log.info("result after validation: {}",rulesUpdateRequest); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 54670ba4..e4d023a2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -53,6 +53,7 @@ public class RuleBuilderService { RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); + mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() @@ -62,25 +63,25 @@ public class RuleBuilderService { } }); ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - ruleFileBluePrintExisting.getAllRules() - .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.getAllRules() + .forEach(ruleFileBluePrintExisting::addRule); Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); - mergedRuleFileBlueprint.setImports(uniqueImports.stream() + ruleFileBluePrintExisting.setImports(uniqueImports.stream() .collect(Collectors.joining(""))); Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); - mergedRuleFileBlueprint.setGlobals(uniqueGlobals.stream() + ruleFileBluePrintExisting.setGlobals(uniqueGlobals.stream() .collect(Collectors.joining(""))); - mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() + ruleFileBluePrintExisting.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() .stream(), mergedRuleFileBlueprint.getFunctions() .stream()) .distinct() .toList()); - mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() + ruleFileBluePrintExisting.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() .stream(), mergedRuleFileBlueprint.getDeclarations() .stream()) @@ -88,9 +89,9 @@ public class RuleBuilderService { .toList()); log.info("merged rules: {}", RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)); return RuleMergingResult.builder() - .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) - .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) - .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBlueprint, false, false)) + .addedGlobalsOffset(mergedRuleFileBlueprint.getGlobals().length()) + .addedImportsOffset(mergedRuleFileBlueprint.getImports().length()) .build(); } -- 2.47.2 From f22a7c450eb0c3af3ae238e2bb0bb607c52b9a7d Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 13:19:16 +0100 Subject: [PATCH 056/108] RED-9472: seperation of system rules adding logs for bug investigation --- .../controller/RuleBuilderController.java | 3 ++- .../v1/server/service/RuleBuilderService.java | 25 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 8c452b07..7d0957b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -63,6 +63,7 @@ public class RuleBuilderController implements RuleBuilderResource { } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + log.info("result of validation: " + droolsValidation); droolsValidationResponse = DroolsValidationResponse.builder() .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() .stream() @@ -99,7 +100,7 @@ public class RuleBuilderController implements RuleBuilderResource { } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } - log.info("result after validation: {}",rulesUpdateRequest); + log.info("result after validation: {}", rulesUpdateRequest); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index e4d023a2..d3a7352e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -69,25 +69,24 @@ public class RuleBuilderService { .map(String::trim) .collect(Collectors.toSet()); ruleFileBluePrintExisting.setImports(uniqueImports.stream() - .collect(Collectors.joining(""))); + .collect(Collectors.joining(""))); Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); ruleFileBluePrintExisting.setGlobals(uniqueGlobals.stream() - .collect(Collectors.joining(""))); + .collect(Collectors.joining(""))); ruleFileBluePrintExisting.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() - .stream(), - mergedRuleFileBlueprint.getFunctions() - .stream()) - .distinct() - .toList()); + .stream(), + mergedRuleFileBlueprint.getFunctions() + .stream()) + .distinct() + .toList()); ruleFileBluePrintExisting.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() - .stream(), - mergedRuleFileBlueprint.getDeclarations() - .stream()) - .distinct() - .toList()); - log.info("merged rules: {}", RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)); + .stream(), + mergedRuleFileBlueprint.getDeclarations() + .stream()) + .distinct() + .toList()); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBlueprint, false, false)) .addedGlobalsOffset(mergedRuleFileBlueprint.getGlobals().length()) -- 2.47.2 From 2d7feb8bfa6cc86d2c3c16087bd727bcdbedfe21 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 13:42:08 +0100 Subject: [PATCH 057/108] RED-9472: seperation of system rules adding logs for bug investigation --- .../redaction/v1/server/controller/RuleBuilderController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 7d0957b9..30aeadc6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -62,6 +62,7 @@ public class RuleBuilderController implements RuleBuilderResource { throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { + log.info("result"+mergingResult.getMergedRules()); var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); log.info("result of validation: " + droolsValidation); droolsValidationResponse = DroolsValidationResponse.builder() -- 2.47.2 From a723fe7129948271e40f660970c5e32aaa62f823 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 14:46:20 +0100 Subject: [PATCH 058/108] RED-9472: seperation of system rules added a method for merging imports to preserve the package description --- .../controller/RuleBuilderController.java | 1 - .../models/RuleFileBluePrint.java | 58 +++++++++++++++++ .../v1/server/service/RuleBuilderService.java | 63 ++++++++++--------- 3 files changed, 92 insertions(+), 30 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 30aeadc6..7d0957b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -62,7 +62,6 @@ public class RuleBuilderController implements RuleBuilderResource { throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { - log.info("result"+mergingResult.getMergedRules()); var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); log.info("result of validation: " + droolsValidation); droolsValidationResponse = DroolsValidationResponse.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 533492e1..196e7f85 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; @@ -34,6 +35,9 @@ public class RuleFileBluePrint { List declarations; List functions; + private static final String IMPORT_PREFIX = "import "; + private static final String PACKAGE_PREFIX = "package "; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -54,6 +58,60 @@ public class RuleFileBluePrint { } + public void addImport(String importStatement) { + + if (importStatement == null || importStatement.trim().isEmpty()) { + return; + } + + String cleanImport = importStatement.trim(); + if (!cleanImport.startsWith(IMPORT_PREFIX)) { + cleanImport = IMPORT_PREFIX + cleanImport; + } + + if (imports == null) { + imports = PACKAGE_PREFIX + "drools\n\n"; + } + + if (!imports.contains(cleanImport)) { + imports = imports.trim() + "\n" + cleanImport; + } + } + + + public void mergeImports(RuleFileBluePrint other) { + + if (other == null || other.getImports() == null) { + return; + } + + addImports(other.getImports()); + } + + + public String getImports() { + + if (imports == null || imports.trim().isEmpty()) { + return PACKAGE_PREFIX + "drools\n\n"; + } + return imports; + } + + + public void addImports(String importStatements) { + + if (importStatements == null || importStatements.trim().isEmpty()) { + return; + } + + Arrays.stream(importStatements.split("\n")) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .filter(line -> !line.startsWith(PACKAGE_PREFIX)) + .forEach(this::addImport); + } + + public boolean removeRuleClassByRuleIdentifier(RuleIdentifier ruleIdentifier) { AtomicBoolean wasRemoved = new AtomicBoolean(false); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index d3a7352e..5c215c2c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -51,10 +51,10 @@ public class RuleBuilderService { public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { - RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); + RuleFileBluePrint mergedRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(existingRules); + RuleFileBluePrint userRulesBluePrint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); - mergedRuleFileBlueprint.getRuleClasses() + userRulesBluePrint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { @@ -62,35 +62,40 @@ public class RuleBuilderService { throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - mergedRuleFileBlueprint.getAllRules() - .forEach(ruleFileBluePrintExisting::addRule); - Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) + + mergedRuleFileBluePrint = removeAllRulesExceptSystemRules(mergedRuleFileBluePrint); + userRulesBluePrint.getAllRules() + .forEach(mergedRuleFileBluePrint::addRule); + + mergedRuleFileBluePrint.mergeImports(userRulesBluePrint); + + Set uniqueGlobals = Arrays.stream((mergedRuleFileBluePrint.getGlobals() + userRulesBluePrint.getGlobals()).replaceAll("\n", "").split("global")) .map(String::trim) + .filter(s -> !s.isEmpty()) .collect(Collectors.toSet()); - ruleFileBluePrintExisting.setImports(uniqueImports.stream() - .collect(Collectors.joining(""))); - Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) - .map(String::trim) - .collect(Collectors.toSet()); - ruleFileBluePrintExisting.setGlobals(uniqueGlobals.stream() - .collect(Collectors.joining(""))); - ruleFileBluePrintExisting.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() - .stream(), - mergedRuleFileBlueprint.getFunctions() - .stream()) - .distinct() - .toList()); - ruleFileBluePrintExisting.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() - .stream(), - mergedRuleFileBlueprint.getDeclarations() - .stream()) - .distinct() - .toList()); + + mergedRuleFileBluePrint.setGlobals(uniqueGlobals.stream() + .map(global -> "global " + global) + .collect(Collectors.joining("\n"))); + + mergedRuleFileBluePrint.setFunctions(Stream.concat(mergedRuleFileBluePrint.getFunctions() + .stream(), + userRulesBluePrint.getFunctions() + .stream()) + .distinct() + .toList()); + + mergedRuleFileBluePrint.setDeclarations(Stream.concat(mergedRuleFileBluePrint.getDeclarations() + .stream(), + userRulesBluePrint.getDeclarations() + .stream()) + .distinct() + .toList()); + return RuleMergingResult.builder() - .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBlueprint, false, false)) - .addedGlobalsOffset(mergedRuleFileBlueprint.getGlobals().length()) - .addedImportsOffset(mergedRuleFileBlueprint.getImports().length()) + .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBluePrint, false, false)) + .addedGlobalsOffset(userRulesBluePrint.getGlobals().length()) + .addedImportsOffset(userRulesBluePrint.getImports().length()) .build(); } -- 2.47.2 From 9cb19fb0f721f1fafeedd0bac1ca1a8fff3df680 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 15:11:38 +0100 Subject: [PATCH 059/108] RED-9472: seperation of system rules added methods for mergin: functions, globals, declarations to RuleFileBluePrint --- .../models/RuleFileBluePrint.java | 61 +++++++++++++++++++ .../v1/server/service/RuleBuilderService.java | 26 +------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 196e7f85..303430c3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -11,6 +12,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -37,6 +39,7 @@ public class RuleFileBluePrint { private static final String IMPORT_PREFIX = "import "; private static final String PACKAGE_PREFIX = "package "; + private static final String GLOBAL_PREFIX = "global "; public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -58,6 +61,64 @@ public class RuleFileBluePrint { } + public void mergeGlobals(RuleFileBluePrint other) { + + if (other == null || other.getGlobals() == null) { + return; + } + + Set existingGlobals = new LinkedHashSet<>(Arrays.stream(this.globals.split("\n")) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .toList()); + + Set newGlobals = Arrays.stream(other.getGlobals().split("\n")) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + StringBuilder globalsBuilder = new StringBuilder(); + Stream.concat(existingGlobals.stream(), newGlobals.stream()) + .distinct() + .forEach(global -> { + if (!global.startsWith(GLOBAL_PREFIX)) { + global = GLOBAL_PREFIX + global; + } + globalsBuilder.append(global).append("\n"); + }); + + this.globals = globalsBuilder.toString().trim(); + } + + + public void mergeFunctions(RuleFileBluePrint other) { + + if (other == null || other.getFunctions() == null) { + return; + } + + this.functions = Stream.concat(this.functions.stream(), + other.getFunctions() + .stream()) + .distinct() + .toList(); + } + + + public void mergeDeclarations(RuleFileBluePrint other) { + + if (other == null || other.getDeclarations() == null) { + return; + } + + this.declarations = Stream.concat(this.declarations.stream(), + other.getDeclarations() + .stream()) + .distinct() + .toList(); + } + + public void addImport(String importStatement) { if (importStatement == null || importStatement.trim().isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 5c215c2c..fa1c91b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -68,29 +68,9 @@ public class RuleBuilderService { .forEach(mergedRuleFileBluePrint::addRule); mergedRuleFileBluePrint.mergeImports(userRulesBluePrint); - - Set uniqueGlobals = Arrays.stream((mergedRuleFileBluePrint.getGlobals() + userRulesBluePrint.getGlobals()).replaceAll("\n", "").split("global")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toSet()); - - mergedRuleFileBluePrint.setGlobals(uniqueGlobals.stream() - .map(global -> "global " + global) - .collect(Collectors.joining("\n"))); - - mergedRuleFileBluePrint.setFunctions(Stream.concat(mergedRuleFileBluePrint.getFunctions() - .stream(), - userRulesBluePrint.getFunctions() - .stream()) - .distinct() - .toList()); - - mergedRuleFileBluePrint.setDeclarations(Stream.concat(mergedRuleFileBluePrint.getDeclarations() - .stream(), - userRulesBluePrint.getDeclarations() - .stream()) - .distinct() - .toList()); + mergedRuleFileBluePrint.mergeGlobals(userRulesBluePrint); + mergedRuleFileBluePrint.mergeFunctions(userRulesBluePrint); + mergedRuleFileBluePrint.mergeDeclarations(userRulesBluePrint); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBluePrint, false, false)) -- 2.47.2 From 88155dc211e96e5852f66a130fbbb9ea7069d8f8 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 15:27:41 +0100 Subject: [PATCH 060/108] RED-9472: seperation of system rules fixed checkstyle --- .../server/rulesmanagement/models/RuleFileBluePrint.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 303430c3..420cc936 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -75,17 +75,14 @@ public class RuleFileBluePrint { Set newGlobals = Arrays.stream(other.getGlobals().split("\n")) .map(String::trim) .filter(line -> !line.isEmpty()) + .map(line -> line.startsWith(GLOBAL_PREFIX) ? line : GLOBAL_PREFIX + line) .collect(Collectors.toCollection(LinkedHashSet::new)); + // Combine globals while maintaining order and format StringBuilder globalsBuilder = new StringBuilder(); Stream.concat(existingGlobals.stream(), newGlobals.stream()) .distinct() - .forEach(global -> { - if (!global.startsWith(GLOBAL_PREFIX)) { - global = GLOBAL_PREFIX + global; - } - globalsBuilder.append(global).append("\n"); - }); + .forEach(globalLine -> globalsBuilder.append(globalLine).append("\n")); this.globals = globalsBuilder.toString().trim(); } -- 2.47.2 From 69ae276c752f77079155f06b2e6b226614dcfeb1 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 18 Dec 2024 09:49:07 +0100 Subject: [PATCH 061/108] RED-9472: seperation of system rules working on fixing bug --- .../redaction/v1/server/controller/RuleBuilderController.java | 3 +-- .../v1/server/rulesmanagement/models/RuleFileBluePrint.java | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 7d0957b9..a61423e9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -63,7 +63,6 @@ public class RuleBuilderController implements RuleBuilderResource { } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); - log.info("result of validation: " + droolsValidation); droolsValidationResponse = DroolsValidationResponse.builder() .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() .stream() @@ -100,7 +99,7 @@ public class RuleBuilderController implements RuleBuilderResource { } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } - log.info("result after validation: {}", rulesUpdateRequest); + log.info("result after validation: {}", rulesUploadResponse); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 420cc936..d2aba3d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -78,7 +78,6 @@ public class RuleFileBluePrint { .map(line -> line.startsWith(GLOBAL_PREFIX) ? line : GLOBAL_PREFIX + line) .collect(Collectors.toCollection(LinkedHashSet::new)); - // Combine globals while maintaining order and format StringBuilder globalsBuilder = new StringBuilder(); Stream.concat(existingGlobals.stream(), newGlobals.stream()) .distinct() -- 2.47.2 From 56bed5a18ce3de4f107cf0a1b9a4f9e466e3d3a7 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 18 Dec 2024 10:47:59 +0100 Subject: [PATCH 062/108] RED-9472: seperation of system rules working on fixing bug --- redaction-service-v1/redaction-service-api-v1/build.gradle.kts | 2 +- .../redaction-service-server-v1/build.gradle.kts | 2 +- .../redaction/v1/server/controller/RuleBuilderController.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 97d4351d..3c05f811 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.649.0-RED9472.0" +val persistenceServiceVersion = "2.651.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 477ca775..750531ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.191.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.649.0-RED9472.0" +val persistenceServiceVersion = "2.651.0-RED9472.0" val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index a61423e9..a54de98a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -99,7 +99,7 @@ public class RuleBuilderController implements RuleBuilderResource { } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } - log.info("result after validation: {}", rulesUploadResponse); + log.info("response: " + new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK)); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } -- 2.47.2 From 6cda5e0eeafa0b0a6a43242d3032f6de589ab323 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Sep 2024 10:24:59 +0200 Subject: [PATCH 063/108] RED-9472: seperation of system rules added endpoints for seperating and readding system rules working on service RED-9472: seperation of system rules refactored endpoints. now removing system rules on download merging user rules with systemrules on upload RED-9472: seperation of system rules RED-9472: seperation of system rules RED-9472: seperation of system rules refactored two have a rulefileblueprint and a rulefileblueprint for validationresult changed the flow of merging and seperating system and user rules added functions and declarations to rule file parsing RED-9472: seperation of system rules RED-9472: seperation of system rules introduced tests working on fixing bug with merging rules RED-9472: seperation of system rules introduced tests RED-9472: seperation of system rules fixed circular depdency RED-9472: seperation of system rules updated persistence service version RED-9472: seperation of system rules refactored rules refactored pcackages RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed some sonarqube errors RED-9472: seperation of system rules fixed pmd error RED-9472: seperation of system rules added deploy stage to gitlab pipeline RED-9472: seperation of system rules RED-9472: seperation of system rules fixed a mistake RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse --- .../redaction-service-api-v1/build.gradle.kts | 3 +- .../v1/resources/RuleBuilderResource.java | 17 +- .../build.gradle.kts | 28 ++- .../controller/RuleBuilderController.java | 96 ++++++++- .../v1/server/model/RuleMergingResult.java | 24 +++ ...ePrint.java => RuleCompilationResult.java} | 67 +++++- .../RuleManagementResources.java | 34 +++ .../DefaultRuleIdentifiersFactory.java | 8 +- .../factory/RuleFileFactory.java | 51 +++-- .../factory/RuleFileParser.java | 82 +++++-- .../migration/RuleFileMigrator.java | 15 +- .../migration/RuleIdentifierMigrator.java | 45 ++-- .../models/ApplicationType.java | 2 +- .../models/BasicDeclaration.java | 34 +++ .../rulesmanagement/models/BasicFunction.java | 36 ++++ .../rulesmanagement}/models/BasicRule.java | 4 +- .../rulesmanagement}/models/OldRule.java | 2 +- .../rulesmanagement}/models/RuleClass.java | 2 +- .../models/RuleFileBluePrint.java | 42 +++- .../models/RuleIdentifier.java | 2 +- .../rulesmanagement}/models/RuleType.java | 2 +- .../rulesmanagement}/models/RuleUnit.java | 2 +- .../rulesmanagement}/utils/RuleFileIO.java | 2 +- .../v1/server/service/RuleBuilderService.java | 169 +++++++++++++++ .../drools/DroolsValidationService.java | 85 ++++---- ....java => RuleCompilationResultParser.java} | 139 +++++++++--- .../all_redact_manager_rules.drl | 0 .../rulesmanagement}/all_rules_documine.drl | 0 .../default_rule_identifiers.txt | 0 .../default_rule_identifiers_dm.txt | 0 .../old_rules_with_translations.csv | 0 .../rulesmanagement}/order_template.txt | 0 .../redaction/v1/server/RuleBuilderTest.java | 149 +++++++++++++ .../services/DroolsUpToDateTest.java | 8 +- .../services/DroolsValidationServiceTest.java | 16 +- .../DroolsCompilationTest.java | 6 +- .../RuleFileBluePrintMergingTest.java | 20 ++ .../RuleFileMigrationTest.java | 4 +- .../factory/RuleFileFactoryTest.java | 10 +- .../EFSA_sanitisation_GFL_v1/rules.txt | 0 .../rulesmanagement/addNewRulesHere.drl | 0 .../rulesmanagement}/demo/table_demo.drl | 0 .../rulesmanagement/dev/Flora/rules.txt | 0 .../docu/EFSA_Regulation_2021/rules.txt | 0 .../docu/EFSA_sanitisation_GFL_v1/rules.txt | 0 .../EFSA_sanitisation_pre_GFL_v1/rules.txt | 0 .../docu/Manual_Redaction/rules.txt | 0 .../rulesmanagement/efsa_regulation_rules.txt | 0 .../userrules/user_rule_update.drl | 50 +++++ .../rules-management/build.gradle.kts | 4 +- .../utility/rules/management/Main.java | 20 +- .../management/RuleManagementResources.java | 56 ----- .../translation/OldRulesParser.java | 203 ------------------ .../RuleFileBluePrintMergingTest.java | 20 -- .../translation/OldRulesParserTest.java | 176 --------------- 55 files changed, 1063 insertions(+), 672 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/{RuleFileBluePrint.java => RuleCompilationResult.java} (61%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/DefaultRuleIdentifiersFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactory.java (68%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileParser.java (50%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleFileMigrator.java (77%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/migration/RuleIdentifierMigrator.java (76%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/ApplicationType.java (73%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/BasicRule.java (80%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/OldRule.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleClass.java (88%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleFileBluePrint.java (85%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleIdentifier.java (97%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleType.java (95%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/models/RuleUnit.java (54%) rename redaction-service-v1/{rules-management/src/main/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/utils/RuleFileIO.java (96%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/{RuleFileParser.java => RuleCompilationResultParser.java} (53%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_redact_manager_rules.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/all_rules_documine.drl (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/default_rule_identifiers_dm.txt (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/old_rules_with_translations.csv (100%) rename redaction-service-v1/{rules-management/src/main/resources => redaction-service-server-v1/src/main/resources/rulesmanagement}/order_template.txt (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/DroolsCompilationTest.java (91%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/RuleFileMigrationTest.java (93%) rename redaction-service-v1/{rules-management/src/test/java/com/knecon/fforesight/utility/rules/management => redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement}/factory/RuleFileFactoryTest.java (94%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl rename redaction-service-v1/{rules-management/src/test/resources => redaction-service-server-v1/src/test/resources/rulesmanagement}/demo/table_demo.drl (100%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java delete mode 100644 redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java delete mode 100644 redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 87ec3822..dc942464 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,11 +4,12 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.631.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" dependencies { implementation("org.springframework:spring-web:6.0.12") implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + api("com.knecon.fforesight:swagger-commons:0.7.0") } publishing { diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java index 3864ba9b..4683fd5f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RuleBuilderResource.java @@ -1,13 +1,26 @@ package com.iqser.red.service.redaction.v1.resources; -import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; - import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; public interface RuleBuilderResource { @PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE) RuleBuilderModel getRuleBuilderModel(); + + @PostMapping(value = "/internal-api/rules/user-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + RulesResponse getRuleFileWithoutSystemRules(@RequestBody SystemRulesSeperationRequest systemRulesSeperationRequest); + + + @PostMapping(value = "/internal-api/rules/system-rules", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + ResponseEntity mergeUserUpdateRules(@RequestBody RulesUpdateRequest rulesUpdateRequest); + } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 262655fd..f3d723ae 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.641.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" @@ -31,10 +31,15 @@ configurations { } } +configurations.all { + resolutionStrategy { + force("com.google.protobuf:protobuf-java:4.27.1") + } +} + dependencies { implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } - implementation(project(":document")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { @@ -47,7 +52,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.50.0") - implementation("com.knecon.fforesight:tenant-commons:0.31.0") + implementation("com.knecon.fforesight:tenant-commons:0.30.0") implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } @@ -56,14 +61,13 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") - implementation("org.ahocorasick:ahocorasick:0.9.0") - implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") - implementation("com.github.roklenarcic:aho-corasick:1.2") + implementation("org.ahocorasick:ahocorasick:0.6.3") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") implementation("org.drools:drools-mvel:${droolsVersion}") implementation("org.kie:kie-spring:7.74.1.Final") + implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.locationtech.jts:jts-core:1.19.0") @@ -82,6 +86,7 @@ dependencies { implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") + api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") @@ -132,7 +137,6 @@ tasks.named("bootBuildImage") { "BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5" ) - environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE! if (project.hasProperty("buildbootDockerHostNetwork")) { @@ -187,19 +191,13 @@ tasks.register("generateJavaDoc", Javadoc::class) { dependsOn("compileJava") dependsOn("delombok") classpath = project.sourceSets["main"].runtimeClasspath - val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { + source = fileTree("${buildDir}/generated/sources/delombok/java/main") { include(droolsImports) } - val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { - include(droolsImports) - } - source = documentFiles + mainFiles - - setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "")) + destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "") options.memberLevel = JavadocMemberLevel.PUBLIC (options as StandardJavadocDocletOptions).apply { title = "API Documentation for Redaction Service ${project.version}" } } - diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 306dccbe..8f3a51dd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,27 +1,111 @@ package com.iqser.red.service.redaction.v1.server.controller; -import java.util.Collections; - +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleBlacklistErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxErrorMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RuleSyntaxWarningMessage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.SystemRulesSeperationRequest; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @RestController +@Slf4j @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { + private final RuleBuilderService ruleBuilderService; + private final DroolsValidationService droolsValidationService; + + @Override public RuleBuilderModel getRuleBuilderModel() { - RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + return this.ruleBuilderService.getRuleBuilderModel(); + } - ruleBuilderModel.setWhenClauses(Collections.emptyList()); - ruleBuilderModel.setThenConditions(Collections.emptyList()); - return ruleBuilderModel; + @Override + public RulesResponse getRuleFileWithoutSystemRules(SystemRulesSeperationRequest systemRulesSeperationRequest) { + + RulesResponse rulesResponse = new RulesResponse(); + String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); + if (filteredRules.isEmpty()) { + throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + } + rulesResponse.setRules(filteredRules); + return rulesResponse; + } + + + @Override + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + + RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); + DroolsValidationResponse droolsValidationResponse; + RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); + if (mergingResult.getMergedRules().isEmpty()) { + throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + } + try { + var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + droolsValidationResponse = DroolsValidationResponse.builder() + .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() + .stream() + .map(droolsSyntaxErrorMessage -> new RuleSyntaxErrorMessage(droolsSyntaxErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxErrorMessage.getColumn(), + droolsSyntaxErrorMessage.getMessage())) + .toList()) + .deprecatedWarnings(droolsValidation.getDeprecatedWarnings() + .stream() + .map(droolsSyntaxDeprecatedWarnings -> new RuleSyntaxWarningMessage(droolsSyntaxDeprecatedWarnings.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsSyntaxDeprecatedWarnings.getColumn(), + droolsSyntaxDeprecatedWarnings.getMessage())) + .toList()) + .blacklistErrorMessages(droolsValidation.getBlacklistErrorMessages() + .stream() + .map(droolsBlacklistErrorMessage -> new RuleBlacklistErrorMessage(droolsBlacklistErrorMessage.getLine() + - (mergingResult.getAddedImportsOffset() + + mergingResult.getAddedGlobalsOffset()), + droolsBlacklistErrorMessage.getColumn(), + droolsBlacklistErrorMessage.getMessage())) + .toList()) + .build(); + if (!droolsValidation.isCompiled()) { + return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + } else { + // + rulesUploadResponse.setRules(mergingResult.getMergedRules()); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + } + } catch (Exception e) { + throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); + } + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); + } + + + private enum RuleFileType { + ENTITY, + COMPONENT } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java new file mode 100644 index 00000000..b2cb1b27 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Schema(description = "Object containing a string of Drools rules.") +public class RuleMergingResult { + + @Schema(description = "The merged rules.") + private String mergedRules; + + @Schema(description = "the length of added imports from sytemRules") + private int addedImportsOffset; + @Schema(description = "the length of added globals from sytemRules") + private int addedGlobalsOffset; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java similarity index 61% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index 3aa1cdef..e96aa697 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -20,13 +20,15 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Builder @NoArgsConstructor @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public final class RuleFileBluePrint { +public final class RuleCompilationResult { String imports; int importLine; @@ -45,9 +47,62 @@ public final class RuleFileBluePrint { } + public void dropRulesByIdentifier(RuleType ruleType) { + + log.info("removing rule with identifier {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropAllRulesExceptSystemRules(List systemRules) { + + log.info("removing all rules except the system rules"); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) + .toList(); + log.info("rules to be removed {}", rulesToBeRemoved); + ruleClasses.removeAll(rulesToBeRemoved); + } + + + public void dropImports() { + + this.imports = ""; + } + + + public void dropQueries() { + + this.queries.clear(); + } + + + public int countRuleOccurences(RuleType ruleType) { + + log.info("counting occurences of files {}", ruleType.name()); + + List rulesToBeRemoved = ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .toList(); + return rulesToBeRemoved.size(); + } + + + private boolean filterOutRule(RuleType ruleType, List filteredRules) { + + return !filteredRules.contains(ruleType.name()); + } + + public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replaceAll("\n", "").split("import")) + return Arrays.stream(imports.replace("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } @@ -76,7 +131,7 @@ public final class RuleFileBluePrint { public List getAllRuleIdentifiers() { return streamAllRules().map(BasicRule::getIdentifier) - .collect(Collectors.toList()); + .toList(); } @@ -96,4 +151,10 @@ public final class RuleFileBluePrint { return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']'; } + + public void addRuleClass(RuleClass ruleClass) { + + this.ruleClasses.add(ruleClass); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java new file mode 100644 index 00000000..331b1e25 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import java.io.InputStream; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; + +@SuppressWarnings("PMD") +public class RuleManagementResources { + + public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + } + return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + } + + + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + + if (applicationType == ApplicationType.RM) { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + } else { + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + } + } + + + public static InputStream getTemplateInputStream() { + + return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java index 2a8de641..c83f7f04 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/DefaultRuleIdentifiersFactory.java @@ -1,12 +1,12 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java similarity index 68% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java index 039664bf..f6332332 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactory.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.Collections.emptySet; @@ -9,13 +9,13 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -48,17 +48,17 @@ public class RuleFileFactory { } RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers); - return buildRuleString(filteredBluePrint); + return buildRuleString(filteredBluePrint, false, false); } @SneakyThrows - public String buildRuleString(RuleFileBluePrint bluePrint) { + public String buildRuleString(RuleFileBluePrint bluePrint, boolean dropImports, boolean dropQueries) { try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) { String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8); List templateRuleOrder = parseRuleOrder(template); - return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder); + return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder, dropImports, dropQueries); } } @@ -72,23 +72,35 @@ public class RuleFileFactory { } - private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder) { + private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder, boolean dropImports, boolean dropQueries) { - Set additionalRuleTypes = bluePrint.ruleClasses() + Set additionalRuleTypes = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleType) .filter(ruleType -> !ruleOrder.contains(ruleType)) .collect(Collectors.toSet()); StringBuilder sb = new StringBuilder(); - sb.append(bluePrint.imports()); + if (!dropImports) { + sb.append(bluePrint.getImports()); + sb.append("\n\n"); + } else { + //todo: this is hacked to enable compiling the rules without imports + sb.append("package drools"); + sb.append("\n\n"); + } + sb.append(bluePrint.getGlobals()); sb.append("\n\n"); - sb.append(bluePrint.globals()); + sb.append("//------------------------------------ declarations ------------------------------------"); sb.append("\n\n"); - sb.append("//------------------------------------ queries ------------------------------------"); - sb.append("\n\n"); - sb.append(bluePrint.queries()); + sb.append(bluePrint.getDeclarations().isEmpty() ? "" : bluePrint.getDeclarations()); sb.append("\n\n"); + if (!dropQueries) { + sb.append("//------------------------------------ queries ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getQueries()); + sb.append("\n\n"); + } for (RuleType ruleBlockType : ruleOrder) { if (ruleBlockType.isWildCard()) { additionalRuleTypes.stream() @@ -101,6 +113,9 @@ public class RuleFileFactory { } writeRuleClass(bluePrint, ruleBlockType, sb); } + sb.append("//------------------------------------ functions ------------------------------------"); + sb.append("\n\n"); + sb.append(bluePrint.getFunctions().isEmpty() ? "" : bluePrint.getFunctions()); return sb.toString().trim() + "\n"; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java similarity index 50% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index 3e17b0db..db0ece77 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -1,30 +1,35 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; +import org.drools.drl.ast.descr.FunctionDescr; import org.drools.drl.ast.descr.ImportDescr; import org.drools.drl.ast.descr.PackageDescr; import org.drools.drl.ast.descr.RuleDescr; import org.drools.drl.parser.DrlParser; import org.kie.internal.builder.conf.LanguageLevelOption; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicDeclaration; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicFunction; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -45,10 +50,19 @@ public class RuleFileParser { @SneakyThrows public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString) { + return buildBluePrintFromRulesString(rulesString, false); + } + + + @SneakyThrows + public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString, boolean removeImports) { + DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); PackageDescr packageDescr = parser.parse(false, rulesString); StringBuilder queryBuilder = new StringBuilder(); List allRules = new LinkedList<>(); + List functions = new LinkedList<>(); + List declarations = new LinkedList<>(); for (RuleDescr rule : packageDescr.getRules()) { if (rule.isQuery()) { queryBuilder.append(rulesString, rule.getStartCharacter(), rule.getEndCharacter()); @@ -57,12 +71,30 @@ public class RuleFileParser { } allRules.add(BasicRule.fromRuleDescr(rule, rulesString)); } - String imports = rulesString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + for (FunctionDescr function : packageDescr.getFunctions()) { + functions.add(BasicFunction.fromFunctionDescr(function, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getEnumDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getClassAndEnumDeclarationDescrs()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + for (AbstractClassTypeDeclarationDescr declaration : packageDescr.getTypeDeclarations()) { + declarations.add(BasicDeclaration.fromDeclarationDescription(declaration, rulesString)); + } + String imports = ""; + if (!removeImports && !packageDescr.getImports().isEmpty()) { + imports = rulesString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -70,7 +102,7 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses); + return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses, declarations, functions); } @@ -89,11 +121,17 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); + if (rulesPerUnit.keySet() != null) { + + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); + } else { + return new ArrayList<>(); + } + } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java similarity index 77% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java index c002fed3..afbdff26 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleFileMigrator.java @@ -1,15 +1,16 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.File; import java.io.FileOutputStream; import java.nio.charset.StandardCharsets; import java.util.List; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -30,7 +31,7 @@ public class RuleFileMigrator { //replaceRules(ruleFileBluePrint, combinedBluePrint); replaceRuleIdentifiers(combinedBluePrint, ruleFileBluePrint); - String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); + String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java similarity index 76% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java index fc8e9026..3889fad1 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/migration/RuleIdentifierMigrator.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.migration; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.migration; import java.io.FileOutputStream; import java.io.IOException; @@ -9,16 +9,15 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleClass; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleClass; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -35,7 +34,6 @@ public class RuleIdentifierMigrator { @SneakyThrows public void migrateAllRuleIdentifiers() { - List parsedRecords = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); RuleFileBluePrint bluePrint = getBluePrint(ApplicationType.RM); // migrateIdentifier(RuleIdentifier.fromString("PII.10.0"), RuleIdentifier.fromString("CBI.20.0"), bluePrint, parsedRecords); @@ -46,20 +44,17 @@ public class RuleIdentifierMigrator { bluePrint = migrateMatchedRuleForAllRules(bluePrint); - String ruleString = RuleFileFactory.buildRuleString(bluePrint); + String ruleString = RuleFileFactory.buildRuleString(bluePrint, false, false); try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) { out.write(ruleString.getBytes(StandardCharsets.UTF_8)); } - String csvString = OldRulesParser.formatAsCsv(parsedRecords); - try (var out = new FileOutputStream("/tmp/old_rules_with_translations.csv")) { - out.write(csvString.getBytes(StandardCharsets.UTF_8)); - } } + //todo: introduce functions and declarations private static RuleFileBluePrint migrateMatchedRuleForAllRules(RuleFileBluePrint bluePrint) { - List migratedRules = bluePrint.ruleClasses() + List migratedRules = bluePrint.getRuleClasses() .stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) @@ -67,7 +62,12 @@ public class RuleIdentifierMigrator { .flatMap(Collection::stream) .map(RuleIdentifierMigrator::migrateMatchedRule) .toList(); - RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.imports(), bluePrint.globals(), bluePrint.queries(), new LinkedList<>()); + RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.getImports(), + bluePrint.getGlobals(), + bluePrint.getQueries(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>()); migratedRules.forEach(migratedBluePrint::addRule); return migratedBluePrint; } @@ -118,7 +118,7 @@ public class RuleIdentifierMigrator { } - public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List records) { + public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint) { BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()) .orElseThrow().findRuleUnitByInteger(oldIdentifier.unit()) @@ -132,9 +132,6 @@ public class RuleIdentifierMigrator { bluePrint.removeRule(oldIdentifier); bluePrint.addRule(newRule); - records.stream() - .filter(record -> record.translatesTo().contains(oldIdentifier)) - .forEach(record -> replaceOldIdentifier(oldIdentifier, newIdentifier, record.translatesTo())); } @@ -152,7 +149,7 @@ public class RuleIdentifierMigrator { } - public record Context(RuleFileBluePrint bluePrint, List records) { + public record Context(RuleFileBluePrint bluePrint) { } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java similarity index 73% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java index 8fdac0d7..53d1122c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/ApplicationType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import lombok.Getter; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java new file mode 100644 index 00000000..4e3acb5c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicDeclaration.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.AbstractClassTypeDeclarationDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicDeclaration { + + @EqualsAndHashCode.Include + String declarationFullTypeName; + String body; + int line; + + + public static BasicDeclaration fromDeclarationDescription(AbstractClassTypeDeclarationDescr declaration, String rulesString) { + + String declarationFullTypeName = declaration.getFullTypeName(); + String body = rulesString.substring(declaration.getStartCharacter(), declaration.getEndCharacter()); + int line = declaration.getLine(); + + return new BasicDeclaration(declarationFullTypeName, body, line); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java new file mode 100644 index 00000000..1793d589 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicFunction.java @@ -0,0 +1,36 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; + +import org.drools.drl.ast.descr.FunctionDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicFunction { + + @EqualsAndHashCode.Include + String functionName; + String returnType; + String body; + int line; + + + public static BasicFunction fromFunctionDescr(FunctionDescr function, String rulesString) { + + String functionName = function.getName(); + String returnType = function.getReturnType(); + String body = rulesString.substring(function.getStartCharacter(), function.getEndCharacter()); + int line = function.getLine(); + + return new BasicFunction(functionName, returnType, body, line); + } + +} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java similarity index 80% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java index 9d77bbb6..cc5c0fe6 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/BasicRule.java @@ -1,8 +1,8 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; public record BasicRule(RuleIdentifier identifier, String name, String code) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java index a6cdb0ab..bb0e2d79 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/OldRule.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import org.drools.drl.ast.descr.RuleDescr; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java similarity index 88% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java index 40468cc0..e70acc0f 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleClass.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.LinkedList; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java similarity index 85% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 2b2ae91f..28b2663c 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Collection; import java.util.Collections; @@ -10,7 +10,29 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -public record RuleFileBluePrint(String imports, String globals, String queries, List ruleClasses) { +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RuleFileBluePrint { + + String imports; + String globals; + String queries; + List ruleClasses; + List declarations; + List functions; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -24,7 +46,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, ruleClass.ruleUnits().remove(ruleUnit); } if (ruleClass.ruleUnits().isEmpty()) { - ruleClasses().remove(ruleClass); + this.ruleClasses.remove(ruleClass); } })); return wasRemoved.get(); @@ -41,7 +63,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRules() { - return ruleClasses().stream() + return this.ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -101,7 +123,7 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public Set getAllRuleIdentifiers() { - return ruleClasses().stream() + return ruleClasses.stream() .map(RuleClass::ruleUnits) .flatMap(Collection::stream) .map(RuleUnit::rules) @@ -122,10 +144,16 @@ public record RuleFileBluePrint(String imports, String globals, String queries, } + public void setImports(String newImports) { + + this.imports = newImports; + } + + public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>()); - ruleClasses().stream() + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) .flatMap(ruleUnit -> ruleUnit.rules() diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java similarity index 97% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java index 569412e4..3fce64a7 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleIdentifier.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Arrays; import java.util.Objects; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java similarity index 95% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java index ffddc766..fcacf44d 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleType.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.Map; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java similarity index 54% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java index ec23ad27..a2b84ba3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleUnit.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleUnit.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.models; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.List; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java similarity index 96% rename from redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java index 3d3ccac6..8efecd93 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/utils/RuleFileIO.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.utils; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.utils; import java.io.File; import java.io.FileInputStream; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java new file mode 100644 index 00000000..32aaf4fc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -0,0 +1,169 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class RuleBuilderService { + + //todo: make this configurable + private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), + new RuleType("MAN"), + new RuleType("X"), + new RuleType("DICT"), + new RuleType("FA"), + new RuleType("LDS"))); + + + public RuleBuilderModel getRuleBuilderModel() { + + RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + + ruleBuilderModel.setWhenClauses(Collections.emptyList()); + ruleBuilderModel.setThenConditions(Collections.emptyList()); + + return ruleBuilderModel; + } + + + public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + log.info("Starting to remove system rules from ruleFile"); + removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); + log.info("Finished removing system rules for ruleFile"); + + return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); + } + + + public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { + + log.info("starting to merge user rules update with system rules"); + RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + mergedRuleFileBlueprint.getRuleClasses() + .forEach(ruleClass -> { + if (systemRules.stream() + .map(RuleType::name) + .toList().contains(ruleClass.ruleType().name())) { + throw new RuntimeException("No system rule updates allowed in user rule update."); + } + }); + removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getRuleClasses() + .stream() + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .flatMap(ruleUnit -> ruleUnit.rules() + .stream())) + .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); + mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() + .stream(), + mergedRuleFileBlueprint.getFunctions() + .stream()) + .toList()); + mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() + .stream(), + mergedRuleFileBlueprint.getDeclarations() + .stream()) + .toList()); + log.info("finished merging user rules update with system rules"); + RuleMergingResult mergingResult = RuleMergingResult.builder() + .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) + .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) + .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .build(); + + return mergingResult; + } + + + private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { + + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + removeAllRulesExceptSystemRules(ruleFileBluePrint); + ruleFileBluePrint.getRuleClasses() + .forEach(ruleClass -> { + if (!systemRuleNames.contains(ruleClass.ruleType().name())) { + throw new RuntimeException("there was an error removing all rules except system rules"); + } + }); + } + + + private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + + List rules = new ArrayList(); + Set systemRuleNames = systemRules.stream() + .map(RuleType::name) + .collect(Collectors.toSet()); + + ruleFileBluePrint.getRuleClasses() + .stream() + .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream() + .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) + .forEach(rule -> rules.add(rule)); + rules.forEach(ruleIdentifier -> { + ruleFileBluePrint.removeRule(ruleIdentifier); + }); + } + + + private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { + + removeSystemRules(ruleFileBluePrint); + for (RuleType systemRule : systemRules) { + List remainingSystemRules = new ArrayList(); + ruleFileBluePrint.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(remainingSystemRules::add)); + if (!remainingSystemRules.isEmpty()) { + throw new RuntimeException("There was an error removing the system rules from the file"); + } + } + } + + + private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { + + for (RuleType systemRule : systemRules) { + List rules = new ArrayList(); + ruleFileBluePrintExisting.findRuleClassByType(systemRule) + .ifPresent(ruleClass -> ruleClass.ruleUnits() + .stream() + .forEach(rules::add)); + rules.forEach(ruleUnit -> { + ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); + }); + ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index 9b983752..e27361be 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -69,51 +69,64 @@ public class DroolsValidationService { private DroolsValidation buildCustomDroolsValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException { - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation(); + DroolsValidation customValidation = ruleCompilationResult.getDroolsValidation(); - addSyntaxDeprecatedWarnings(ruleFileBluePrint, customValidation); + addSyntaxDeprecatedWarnings(ruleCompilationResult, customValidation); - addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation); + addSyntaxErrorMessages(ruleFileType, ruleCompilationResult, customValidation); if (redactionServiceSettings.isRuleExecutionSecured()) { - addBlacklistErrorMessages(ruleFileBluePrint, customValidation); + addBlacklistErrorMessages(ruleCompilationResult, customValidation); } return customValidation; } - private void addSyntaxDeprecatedWarnings(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxDeprecatedWarnings(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { // find deprecated elements in the ruleFileBluePrint - DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint); + DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleCompilationResult); if (warningMessageForImports != null) { customValidation.getDeprecatedWarnings().add(warningMessageForImports); } + customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleCompilationResult)); + } + + + private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleCompilationResult ruleCompilationResult) { + + return DroolsSyntaxDeprecatedWarnings.builder() + .message("global ComponentMappingService " + + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL + + "\n is missing from the rules, consider adding it, as it will be required in future versions!") + .line(ruleCompilationResult.getGlobalsLine()) + .column(0) + .build(); customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } - private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { + private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleCompilationResult ruleCompilationResult) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { - String imports = ruleFileBluePrint.getImports(); + String imports = ruleCompilationResult.getImports(); SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation(); List matches = classesSearchImplementation.getMatches(imports); if (!matches.isEmpty()) { String sb = "Following imports are deprecated: \n" + matches.stream() .map(m -> imports.substring(m.startIndex(), m.endIndex())) .collect(Collectors.joining("\n")); - return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build(); + return DroolsSyntaxDeprecatedWarnings.builder().line(ruleCompilationResult.getImportLine()).column(0).message(sb).build(); } } return null; } - private List getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { + private List getWarningsForDeprecatedRules(RuleCompilationResult ruleCompilationResult) { List warningMessages = new ArrayList<>(); @@ -122,7 +135,7 @@ public class DroolsValidationService { SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation(); Map deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap(); - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { List matches = methodsSearchImplementation.getMatches(basicRule.getCode()); @@ -146,32 +159,32 @@ public class DroolsValidationService { } - private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addSyntaxErrorMessages(RuleFileType ruleFileType, RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { - RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { - case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); - case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + RuleCompilationResult baseRuleCompilationResult = switch (ruleFileType) { + case ENTITY -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); + case COMPONENT -> RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseComponentRuleFileString()); }; - if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) { + if (!importsAreValid(baseRuleCompilationResult, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getImportLine()) + .line(ruleCompilationResult.getImportLine()) .column(0) - .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports())) + .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleCompilationResult.getImports())) .build()); } - if (!ruleFileBluePrint.getGlobals().contains(baseRuleFileBluePrint.getGlobals())) { + if (!ruleCompilationResult.getGlobals().contains(baseRuleCompilationResult.getGlobals())) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() - .line(ruleFileBluePrint.getGlobalsLine()) + .line(ruleCompilationResult.getGlobalsLine()) .column(0) - .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals())) + .message(String.format("Removing the globals is not allowed! Must be: %n%s", baseRuleCompilationResult.getGlobals())) .build()); } - baseRuleFileBluePrint.getQueries() + baseRuleCompilationResult.getQueries() .forEach(basicQuery -> { - if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { + if (!validateQueryIsPresent(basicQuery, ruleCompilationResult)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(basicQuery.getLine()) @@ -182,7 +195,7 @@ public class DroolsValidationService { }); if (ruleFileType.equals(RuleFileType.ENTITY)) { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; - if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { + if (!validateAgendaGroupIsPresent(ruleCompilationResult, requiredAgendaGroup)) { customValidation.getSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() .line(0) @@ -194,18 +207,18 @@ public class DroolsValidationService { } - private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) { + private boolean validateAgendaGroupIsPresent(RuleCompilationResult ruleCompilationResult, String agendaGroupName) { - return ruleFileBluePrint.streamAllRules() + return ruleCompilationResult.streamAllRules() .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName)); } - private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) { + private boolean importsAreValid(RuleCompilationResult baseRuleCompilationResult, RuleCompilationResult ruleCompilationResult) { // imports may shrink, but not add anything new! - Set baseImports = baseRuleFileBluePrint.getImportSplitByKeyword(); - Set imports = ruleFileBluePrint.getImportSplitByKeyword(); + Set baseImports = baseRuleCompilationResult.getImportSplitByKeyword(); + Set imports = ruleCompilationResult.getImportSplitByKeyword(); Set additionalImports = Sets.difference(imports, baseImports); @@ -214,15 +227,15 @@ public class DroolsValidationService { } - private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) { + private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleCompilationResult ruleCompilationResult) { - return ruleFileBluePrint.getQueries() + return ruleCompilationResult.getQueries() .stream() .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode())); } - private void addBlacklistErrorMessages(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) { + private void addBlacklistErrorMessages(RuleCompilationResult ruleCompilationResult, DroolsValidation customValidation) { List blacklistErrorMessages = new ArrayList<>(); @@ -234,14 +247,14 @@ public class DroolsValidationService { // check also the imports DroolsBlacklistErrorMessage blacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, - ruleFileBluePrint.getImports(), - ruleFileBluePrint.getImportLine()); + ruleCompilationResult.getImports(), + ruleCompilationResult.getImportLine()); if (blacklistErrorMessage != null) { blacklistErrorMessages.add(blacklistErrorMessage); } // check the rules - for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) { + for (RuleClass ruleClass : ruleCompilationResult.getRuleClasses()) { for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { for (BasicRule basicRule : ruleUnit.rules()) { DroolsBlacklistErrorMessage ruleBlacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java similarity index 53% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index cd6a953e..616cabde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -22,21 +22,48 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.RuleType; import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @UtilityClass -public class RuleFileParser { +public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { + + DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); + List allRules = new LinkedList<>(); + List allQueries = new LinkedList<>(); + PackageDescr packageDescr = new PackageDescr(); + String imports = ""; + String globals = ""; + List ruleClasses = buildRuleClasses(allRules); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + @SneakyThrows - public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); @@ -52,12 +79,15 @@ public class RuleFileParser { } } - String imports = ruleString.substring(0, - packageDescr.getImports() - .stream() - .mapToInt(ImportDescr::getEndCharacter) - .max() - .orElseThrow() + 1); + String imports = ""; + if (!removedImports) { + imports = ruleString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); + } String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -65,21 +95,81 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); - return new RuleFileBluePrint(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); + return new RuleCompilationResult(imports.trim(), + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsValidation); + } + + + @SneakyThrows + public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { + + return buildRuleCompilationResultFromRuleString(ruleString,false); + } + + + @SneakyThrows + public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { + + StringBuilder ruleStringBuilder = new StringBuilder(); + + // Append imports + ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); + + // Append globals + if (!bluePrint.getGlobals().isEmpty()) { + ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); + } + + // Append queries + for (BasicQuery query : bluePrint.getQueries()) { + ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); + } + + // Append rules + for (RuleClass ruleClass : bluePrint.getRuleClasses()) { + ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); + } + + // Return the final rule string + return ruleStringBuilder.toString().trim(); + } + + + private String buildQueryString(BasicQuery query) { + + return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; + } + + + private String buildRuleString(RuleClass ruleClass) { + + StringBuilder ruleBuilder = new StringBuilder(); + + // Use RuleType to distinguish between different rule types, if needed + ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); + + for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { + for (BasicRule rule : ruleUnit.rules()) { + // Assuming BasicRule has a method to retrieve the condition as a string + ruleBuilder.append(rule.getCode()).append("\n"); + } + } + ruleBuilder.append("end"); + + return ruleBuilder.toString(); } @@ -126,6 +216,7 @@ public class RuleFileParser { private List buildRuleClasses(List allRules) { + //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_redact_manager_rules.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/all_rules_documine.drl diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/default_rule_identifiers_dm.txt diff --git a/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/old_rules_with_translations.csv diff --git a/redaction-service-v1/rules-management/src/main/resources/order_template.txt b/redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt similarity index 100% rename from redaction-service-v1/rules-management/src/main/resources/order_template.txt rename to redaction-service-v1/redaction-service-server-v1/src/main/resources/rulesmanagement/order_template.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java new file mode 100644 index 00000000..1996af83 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -0,0 +1,149 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RuleBuilderTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); + private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); + private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); + + @Autowired + RuleBuilderService ruleBuilderService; + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import(LayoutParsingServiceProcessorConfiguration.class) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RuleBuilderTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + public void removeSystemRulesTest() { + + String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); + String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + + Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + + Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + } + + + @Test + public void mergeUserRulesUpdateTest() { + + RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); + String mergedRules = mergingResult.getMergedRules(); + RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); + ruleFileBluePrintUserRulesUpdate.getRuleClasses() + .forEach(ruleClass -> { + Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); + }); + + } + + + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getRuleClasses() + .stream() + .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) + .collect(Collectors.toList()).size() == 0; + } + + + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getImports().isEmpty(); + } + + + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + + return ruleCompilationResult.getQueries().isEmpty(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java index cac2af47..2d95017d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java @@ -11,10 +11,10 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.core.io.ClassPathResource; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.BasicRule; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index 4db674a4..0bd9ecec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -22,10 +22,10 @@ import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; -import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; import lombok.SneakyThrows; @@ -232,11 +232,11 @@ class DroolsValidationServiceTest { if (droolsValidation.isCompiled()) { continue; } - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(rulesString); + RuleCompilationResult baseRuleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(RuleManagementResources.getBaseRuleFileString()); - rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); - rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + rulesString = rulesString.replace(ruleCompilationResult.getImports(), baseRuleCompilationResult.getImports()); + rulesString = rulesString.replace(ruleCompilationResult.getGlobals(), baseRuleCompilationResult.getGlobals()); try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); @@ -457,9 +457,9 @@ class DroolsValidationServiceTest { end """; - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + RuleCompilationResult ruleCompilationResult = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(ruleString); - assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled()); + assertFalse(ruleCompilationResult.getDroolsValidation().isCompiled()); } } \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java similarity index 91% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java index f8d25b6a..fa77a92b 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/DroolsCompilationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,10 +27,10 @@ public class DroolsCompilationTest { @Test public void testValidateRuleSyntax() throws IOException { - URL rmURL = Resources.getResource("all_redact_manager_rules.drl"); + URL rmURL = Resources.getResource("rulesmanagement/all_redact_manager_rules.drl"); String rmRule = Resources.toString(rmURL, StandardCharsets.UTF_8); - URL dmURL = Resources.getResource("all_rules_documine.drl"); + URL dmURL = Resources.getResource("rulesmanagement/all_rules_documine.drl"); String dmRule = Resources.toString(dmURL, StandardCharsets.UTF_8); assertTrue(validateRuleSyntax(rmRule).isValid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java new file mode 100644 index 00000000..577f996a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileBluePrintMergingTest.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.rulesmanagement; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; + +public class RuleFileBluePrintMergingTest { + + @Test + public void testBothRuleFilesCanBeMerged() { + + RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); + assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); + } + +} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java similarity index 93% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java index c3629d9c..be926def 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleFileMigrationTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management; +package com.iqser.red.service.redaction.v1.server.rulesmanagement; import java.io.File; import java.nio.file.Files; @@ -8,7 +8,7 @@ import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.migration.RuleFileMigrator; import lombok.SneakyThrows; diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java similarity index 94% rename from redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java index b6b85cef..ce84ebbf 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileFactoryTest.java @@ -1,4 +1,4 @@ -package com.knecon.fforesight.utility.rules.management.factory; +package com.iqser.red.service.redaction.v1.server.rulesmanagement.factory; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -19,10 +19,10 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/EFSA_sanitisation_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/addNewRulesHere.drl new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl similarity index 100% rename from redaction-service-v1/rules-management/src/test/resources/demo/table_demo.drl rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/demo/table_demo.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dev/Flora/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/dossier_templates_v2/docu/Manual_Redaction/rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/efsa_regulation_rules.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl new file mode 100644 index 00000000..16ea73d2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/rulesmanagement/userrules/user_rule_update.drl @@ -0,0 +1,50 @@ +package drools +//------------------------------------ Table extraction rules ------------------------------------ + +// Rule unit: TAB.0 +rule "TAB.0.0: Changed Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Changed Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +// Rule unit: TAB.6 +rule "TAB.6.1: Changed Targeted cell extraction (Experimental Stop date)" + when + $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2")) + $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList() + TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList() + $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList() + then + entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date")); + end + diff --git a/redaction-service-v1/rules-management/build.gradle.kts b/redaction-service-v1/rules-management/build.gradle.kts index 2aaea40c..b005ed7e 100644 --- a/redaction-service-v1/rules-management/build.gradle.kts +++ b/redaction-service-v1/rules-management/build.gradle.kts @@ -27,10 +27,10 @@ sourceSets { } dependencies { - implementation(project(":redaction-service-server-v1")) + implementation(project(mapOf("path" to ":redaction-service-server-v1"))) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") - + implementation(project(":redaction-service-server-v1")) implementation("com.github.javaparser:javaparser-core:3.25.3") implementation("org.drools:drools-drl-parser:8.41.0.Final") implementation("org.apache.commons:commons-csv:1.10.0") diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java index 7267bd20..6ab58cf3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java @@ -17,12 +17,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import lombok.SneakyThrows; @@ -72,7 +71,6 @@ public class Main { Set identifiers = switch (type) { case "list", "l" -> RuleIdentifier.fromListOfIdentifiersString(input); case "file", "f" -> RuleFileParser.parseRuleIdentifiersFromFile(input); - case "old", "o" -> OldRulesParser.translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(input), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid", cmd.getOptionValue("t"))); }; ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType); @@ -102,7 +100,6 @@ public class Main { String type = cmd.hasOption("t") ? cmd.getOptionValue("t") : "f"; Map> identifiersPerFile = switch (type) { case "file", "f" -> parseIdentifiersFromFiles(Path.of(cmd.getOptionValue("input"))); - case "old", "o" -> translateIdentifiersFromOldFiles(Path.of(cmd.getOptionValue("input")), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid in combination with the \"recursive\" flag", cmd.getOptionValue("t"))); }; identifiersPerFile.keySet() @@ -137,13 +134,6 @@ public class Main { } - @SneakyThrows - private static Map> translateIdentifiersFromOldFiles(Path inputDirectory, ApplicationType applicationType) { - - return RuleFileIO.streamAllRuleFilesInDirectory(inputDirectory) - .collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, applicationType))); - } - @SneakyThrows private static Map> parseIdentifiersFromFiles(Path inputDirectory) { diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java deleted file mode 100644 index af844a6d..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -public class RuleManagementResources { - - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_redact_manager_rules.drl"); - } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl"); - } - - - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { - - if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers.txt"); - } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers_dm.txt"); - } - } - - - public static InputStream getTemplateInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("order_template.txt"); - } - - - public static InputStream getOldRulesCsvInputStream() { - - return RuleManagementResources.class.getClassLoader().getResourceAsStream("old_rules_with_translations.csv"); - } - - - @SneakyThrows - public static String createTempOldRulesCsv(String formattedAsCsv) { - - File csvFile = File.createTempFile("old_rules_with_translations-", ".csv"); - try (var out = new FileOutputStream(csvFile)) { - out.write(formattedAsCsv.getBytes(StandardCharsets.UTF_8)); - } - return csvFile.toString(); - } - -} diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java deleted file mode 100644 index ae0f3134..00000000 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java +++ /dev/null @@ -1,203 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; -import org.drools.drl.ast.descr.PackageDescr; -import org.drools.drl.ast.descr.RuleDescr; -import org.drools.drl.parser.DrlParser; -import org.kie.internal.builder.conf.LanguageLevelOption; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.OldRule; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; - -@UtilityClass -public class OldRulesParser { - - static List HEADERS = List.of("id", "old rule names", "old rule code", "translates to"); - - - @SneakyThrows - public void createSetOfTranslatedRuleIdentifiersForEachFileInTheCsv() { - - List allHeaders = List.of(); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - - } - - - public Set translateOldRulesFileToNewIdentifiers(File oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesFileToNewIdentifiers(oldRulesFile.toString(), applicationType); - } - - - public Set translateOldRulesFileToNewIdentifiers(String oldRulesFile, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(oldRulesFile), applicationType); - } - - - public Set translateEscapedOldRulesStringToNewIdentifiers(String escapedOldRulesString, ApplicationType applicationType) { - - return translateOldRulesStringToNewIdentifiers(RuleFileIO.unescapeAndUnWrap(escapedOldRulesString), applicationType); - } - - - public Set translateOldRulesStringToNewIdentifiers(String oldRulesString, ApplicationType applicationType) { - - List oldRules = parseOldRules(oldRulesString); - List records = getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - Map> translationPairs = new HashMap<>(); - for (OldRule oldRule : oldRules) { - List translatedIdentifiers = records.stream() - .filter(oldRulesCsvRecord -> oldRulesCsvRecord.code().equals(oldRule.code())) - .map(OldRulesCsvRecord::translatesTo) - .findAny() - .orElse(Collections.emptyList()); - translationPairs.put(oldRule, translatedIdentifiers); - } - boolean allTranslated = true; - for (OldRule oldRule : translationPairs.keySet()) { - if (translationPairs.get(oldRule).isEmpty()) { - allTranslated = false; - records.add(new OldRulesCsvRecord(records.size(), oldRule.code(), oldRule.name(), Collections.emptyList())); - System.out.printf( - "Rule %s has not been translated yet! %nIt has been added to the bottom of the old_rules_with_translations.csv file. To continue, translate the rule and enter its identifier in the csv, then try again!%n", - oldRule.name()); - } - } - if (!allTranslated) { - String formattedAsCsv = formatAsCsv(records); - String fileLocation = RuleManagementResources.createTempOldRulesCsv(formattedAsCsv); - System.out.printf("CSV File with updated values is located at %s%n", fileLocation); - throw new IllegalArgumentException("Non translated Rule found!"); - } - RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); - - translationPairs.forEach((key, value) -> System.out.printf("Rule %s will be translated to %s \n", - key.name(), - String.format("%s: %s", - value, - value.stream() - .map(bluePrint::findRuleByIdentifier) - .flatMap(Collection::stream) - .map(BasicRule::name) - .toList()))); - - return Stream.concat(// - Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),// - translationPairs.values() - .stream() - .flatMap(Collection::stream)) - .map(ruleIdentifier -> new RuleIdentifier(ruleIdentifier.type(), ruleIdentifier.unit(), null)) - .collect(Collectors.toSet()); - } - - - @SneakyThrows - public String formatAsCsv(List records) { - - StringWriter sw = new StringWriter(); - CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS.toArray(String[]::new)).build(); - try (CSVPrinter printer = new CSVPrinter(sw, csvFormat)) { - for (OldRulesCsvRecord record : records) { - printer.printRecord(Stream.of(record.id, RuleFileIO.escapeAndWrap(record.names), RuleFileIO.escapeAndWrap(record.code), record.translatesTo)); - } - } - return sw.toString(); - } - - - @SneakyThrows - private List parseOldRules(String oldRuleString) { - - DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); - PackageDescr packageDescr = parser.parse(false, oldRuleString); - List oldRules = new LinkedList<>(); - for (RuleDescr rule : packageDescr.getRules()) { - if (!rule.isRule()) { - continue; - } - oldRules.add(OldRule.fromRuleDescr(rule, oldRuleString)); - } - return oldRules; - } - - - @SneakyThrows - public List getOldRulesCsvRecords(InputStream allRulesCsvInputStream) { - - List parsedRecords = new LinkedList<>(); - try (Reader reader = new InputStreamReader(allRulesCsvInputStream)) { - Iterable records = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord().parse(reader); - for (CSVRecord record : records) { - String[] values = record.values(); - parsedRecords.add(new OldRulesCsvRecord(Long.parseLong(values[0]), - RuleFileIO.unescapeAndUnWrap(values[2]), - RuleFileIO.unescapeAndUnWrap(values[1]), - parseRuleIdentifiers(values[3]))); - } - } - return parsedRecords; - } - - - private List parseRuleIdentifiers(String value) { - - String cleanedValue = value; - if (cleanedValue.startsWith("[")) { - cleanedValue = cleanedValue.substring(1); - } - if (cleanedValue.endsWith("]")) { - cleanedValue = cleanedValue.substring(0, cleanedValue.length() - 1); - } - if (cleanedValue.isEmpty() || cleanedValue.isBlank()) { - return Collections.emptyList(); - } - List ruleIdentifiers = new LinkedList<>(); - for (String identifier : cleanedValue.split(", ")) { - ruleIdentifiers.add(RuleIdentifier.fromString(identifier)); - } - return ruleIdentifiers; - } - - - private List parseBooleanList(String[] values) { - - return Arrays.stream(values) - .map(Boolean::parseBoolean) - .toList(); - } - - - public record OldRulesCsvRecord(long id, String code, String names, List translatesTo) { - - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java deleted file mode 100644 index 559101ac..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.knecon.fforesight.utility.rules.management; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; - -public class RuleFileBluePrintMergingTest { - - @Test - public void testBothRuleFilesCanBeMerged() { - - RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles(); - assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size()); - } - -} diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java deleted file mode 100644 index b78ab534..00000000 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package com.knecon.fforesight.utility.rules.management.translation; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import com.knecon.fforesight.utility.rules.management.RuleManagementResources; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; -import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; -import com.knecon.fforesight.utility.rules.management.models.ApplicationType; -import com.knecon.fforesight.utility.rules.management.models.BasicRule; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; -import com.knecon.fforesight.utility.rules.management.models.RuleType; -import com.knecon.fforesight.utility.rules.management.models.RuleUnit; -import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; - -import lombok.SneakyThrows; - -@SuppressWarnings("PMD") -class OldRulesParserTest { - - @Test - @SneakyThrows - @Disabled - public void translateOldRulesToNewIdentifiersTest() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("efsa_regulation_rules.txt")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateEscapedOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @SneakyThrows - @Disabled - public void translateMoreOldRules() { - - try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("basf/demo/table_demo.drl")) { - String oldRulesString = new String(inputStream.readAllBytes()); - Set identifiers = OldRulesParser.translateOldRulesStringToNewIdentifiers(oldRulesString, ApplicationType.RM); - System.out.println(identifiers); - } - } - - - @Test - @Disabled - public void printTranslationsTest() { - - List records = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")) - .forEach(type -> { - List rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type) - .orElseThrow().ruleUnits(); - rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records)); - }); - } - - - private void printOldRulesThatTranslatesToNewRule(RuleUnit ruleUnit, List records) { - - if (ruleUnit.rules().isEmpty()) { - System.out.println("Rule unit empty, skipping!\n"); - return; - } - RuleIdentifier identifier = ruleUnit.rules() - .get(0).identifier(); - RuleIdentifier unitIdentifier = new RuleIdentifier(identifier.type(), identifier.unit(), null); - String oldNames = records.stream() - .filter(r -> r.translatesTo() - .stream() - .anyMatch(i -> i.matches(unitIdentifier))) - .map(OldRulesParser.OldRulesCsvRecord::names) - .map(OldRulesParserTest::removeIdFromName) - .distinct() - .collect(Collectors.joining(", ")); - System.out.println(unitIdentifier.toRuleUnitString() + " " + ruleUnit.rules() - .stream() - .map(BasicRule::name) - .collect(Collectors.joining(", "))); - System.out.println("translate from"); - System.out.println(oldNames); - System.out.println(); - System.out.println(); - } - - - private static String removeIdFromName(String name) { - - String[] values = name.split(":"); - return String.join(":", Arrays.copyOfRange(values, 1, values.length)); - } - - - @Test - @Disabled - @SneakyThrows - public void findAllCustomerRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("pilot/EFSA_sanitisation_GFL_v1").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.RM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.RM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void findAllRuleFilesInDossierTemplatesRepoAndTranslate() { - - String dossierTemplatesRepo = getClass().getClassLoader().getResource("business-logic/Syngenta_RSS").getFile(); - RuleFileIO.streamAllRuleFilesInDirectory(Path.of(dossierTemplatesRepo)) - .map(e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, ApplicationType.DM)) - .map(e -> RuleFileFactory.createFileFromIdentifiers(e, ApplicationType.DM)) - .peek(System.out::println) - .map(RuleFileIO::escapeAndWrap) - .forEach(System.out::println); - - } - - - @Test - @Disabled - @SneakyThrows - public void translateDossierTemplatesV2() { - -// String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/RED/dossier-templates-v2/"; -// Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), Files.walk(Path.of(dossierTemplatesRepo + "docu")), Files.walk(Path.of(dossierTemplatesRepo + "qa"))) - String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/"; - Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), - Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) - // - ) - .flatMap(Function.identity())// - .filter(path -> path.getFileName().toString().equals("rules.drl"))// - .map(Path::toFile)// - .forEach(this::translateOldRuleFile); - - } - - - @SneakyThrows - private void translateOldRuleFile(File oldRulesFile) { - - Set identifiers = OldRulesParser.translateOldRulesFileToNewIdentifiers(oldRulesFile, ApplicationType.RM); - String newRulesString = RuleFileFactory.createFileFromIdentifiers(identifiers, ApplicationType.RM); - String result = RuleFileIO.escapeAndWrap(newRulesString); - - try (var out = new FileOutputStream(oldRulesFile)) { - out.write(result.getBytes(StandardCharsets.UTF_8)); - } - } - -} \ No newline at end of file -- 2.47.2 From bbd2ad60c240a02edd87f9f40513037b71d43746 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 09:44:51 +0100 Subject: [PATCH 064/108] RED-9472: seperation of system rules code review changes --- .../controller/RuleBuilderController.java | 9 +- .../model/drools/RuleCompilationResult.java | 46 +---- .../RuleManagementResources.java | 19 +- .../factory/RuleFileParser.java | 15 +- .../models/RuleFileBluePrint.java | 28 +++ .../v1/server/service/RuleBuilderService.java | 91 ++------- .../drools/RuleCompilationResultParser.java | 83 +------- .../redaction/v1/server/RuleBuilderTest.java | 183 ++++++++---------- 8 files changed, 147 insertions(+), 327 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 8f3a51dd..aa215fc7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -45,7 +45,7 @@ public class RuleBuilderController implements RuleBuilderResource { RulesResponse rulesResponse = new RulesResponse(); String filteredRules = this.ruleBuilderService.cleanRuleFileOfSystemRules(systemRulesSeperationRequest.getRules(), true); if (filteredRules.isEmpty()) { - throw new RuntimeException("There was an error when cleaning the rulefile of sytem rules"); + throw new AssertionError("There was an error when cleaning the rulefile of sytem rules"); } rulesResponse.setRules(filteredRules); return rulesResponse; @@ -53,13 +53,13 @@ public class RuleBuilderController implements RuleBuilderResource { @Override - public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { + public ResponseEntity mergeUserUpdateRules(RulesUpdateRequest rulesUpdateRequest) { RulesUploadResponse rulesUploadResponse = new RulesUploadResponse(); DroolsValidationResponse droolsValidationResponse; RuleMergingResult mergingResult = ruleBuilderService.mergeUserRulesAndSystemRules(rulesUpdateRequest.getExistingRules(), rulesUpdateRequest.getUpdatedRules()); if (mergingResult.getMergedRules().isEmpty()) { - throw new RuntimeException("There was an error when merging the user rule update into the rule file"); + throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); @@ -90,7 +90,8 @@ public class RuleBuilderController implements RuleBuilderResource { .toList()) .build(); if (!droolsValidation.isCompiled()) { - return new ResponseEntity<>(droolsValidationResponse, HttpStatus.UNPROCESSABLE_ENTITY); + rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); + return new ResponseEntity<>(rulesUploadResponse, HttpStatus.UNPROCESSABLE_ENTITY); } else { // rulesUploadResponse.setRules(mergingResult.getMergedRules()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java index e96aa697..3506a31f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleCompilationResult.java @@ -49,60 +49,16 @@ public final class RuleCompilationResult { public void dropRulesByIdentifier(RuleType ruleType) { - log.info("removing rule with identifier {}", ruleType.name()); - List rulesToBeRemoved = ruleClasses.stream() .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); ruleClasses.removeAll(rulesToBeRemoved); } - public void dropAllRulesExceptSystemRules(List systemRules) { - - log.info("removing all rules except the system rules"); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> filterOutRule(ruleClass.ruleType(), systemRules)) - .toList(); - log.info("rules to be removed {}", rulesToBeRemoved); - ruleClasses.removeAll(rulesToBeRemoved); - } - - - public void dropImports() { - - this.imports = ""; - } - - - public void dropQueries() { - - this.queries.clear(); - } - - - public int countRuleOccurences(RuleType ruleType) { - - log.info("counting occurences of files {}", ruleType.name()); - - List rulesToBeRemoved = ruleClasses.stream() - .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) - .toList(); - return rulesToBeRemoved.size(); - } - - - private boolean filterOutRule(RuleType ruleType, List filteredRules) { - - return !filteredRules.contains(ruleType.name()); - } - - public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replace("\n", "").split("import")) + return Arrays.stream(imports.replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java index 331b1e25..e423dde8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/RuleManagementResources.java @@ -1,34 +1,37 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement; +import java.io.IOException; import java.io.InputStream; +import org.drools.io.ClassPathResource; + import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.ApplicationType; @SuppressWarnings("PMD") public class RuleManagementResources { - public static InputStream getAllRulesInputStream(ApplicationType applicationType) { + public static InputStream getAllRulesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_redact_manager_rules.drl"); + return new ClassPathResource("drools/all_redact_manager_rules.drl").getInputStream(); } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("drools/all_rules_documine.drl"); + return new ClassPathResource("drools/all_rules_documine.drl").getInputStream(); } - public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) { + public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) throws IOException { if (applicationType == ApplicationType.RM) { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers.txt").getInputStream(); } else { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/default_rule_identifiers_dm.txt"); + return new ClassPathResource("rulesmanagement/default_rule_identifiers_dm.txt").getInputStream(); } } - public static InputStream getTemplateInputStream() { + public static InputStream getTemplateInputStream() throws IOException { - return RuleManagementResources.class.getClassLoader().getResourceAsStream("rulesmanagement/order_template.txt"); + return new ClassPathResource("rulesmanagement/order_template.txt").getInputStream(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java index db0ece77..6354f93c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/factory/RuleFileParser.java @@ -4,7 +4,6 @@ import static java.util.stream.Collectors.groupingBy; import java.io.File; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -121,16 +120,12 @@ public class RuleFileParser { Map> rulesPerUnit = rules.stream() .collect(groupingBy(rule -> rule.identifier().unit())); - if (rulesPerUnit.keySet() != null) { - return rulesPerUnit.keySet() - .stream() - .sorted() - .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) - .collect(Collectors.toList()); - } else { - return new ArrayList<>(); - } + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 28b2663c..ca185b63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; @@ -53,6 +54,33 @@ public class RuleFileBluePrint { } + public boolean removeRuleClassByRuleIdentifier(RuleIdentifier ruleIdentifier) { + + AtomicBoolean wasRemoved = new AtomicBoolean(false); + + findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> { + List unitsToRemove = new ArrayList<>(); + + ruleClass.ruleUnits() + .forEach(ruleUnit -> { + boolean removed = ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier)); + if (removed) { + wasRemoved.set(true); + if (ruleUnit.rules().isEmpty()) { + unitsToRemove.add(ruleUnit); + } + } + }); + + ruleClass.ruleUnits().removeAll(unitsToRemove); + if (ruleClass.ruleUnits().isEmpty()) { + this.ruleClasses.remove(ruleClass); + } + }); + return wasRemoved.get(); + } + + public Optional findRuleClassByType(RuleType ruleType) { return ruleClasses.stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 32aaf4fc..d20e8337 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -3,9 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -16,8 +15,6 @@ import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFil import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleType; -import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleUnit; import lombok.extern.slf4j.Slf4j; @@ -25,13 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - //todo: make this configurable - private final List systemRules = new ArrayList<>(Arrays.asList(new RuleType("AI"), - new RuleType("MAN"), - new RuleType("X"), - new RuleType("DICT"), - new RuleType("FA"), - new RuleType("LDS"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); public RuleBuilderModel getRuleBuilderModel() { @@ -48,28 +44,24 @@ public class RuleBuilderService { public String cleanRuleFileOfSystemRules(String rulesString, boolean removeImports) { RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); - log.info("Starting to remove system rules from ruleFile"); - removeSystemRulesAndCheckIfAnyRemain(ruleFileBluePrint); - log.info("Finished removing system rules for ruleFile"); - + removeSystemRules(ruleFileBluePrint); return RuleFileFactory.buildRuleString(ruleFileBluePrint, removeImports, true); } public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { - log.info("starting to merge user rules update with system rules"); RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() - .map(RuleType::name) - .toList().contains(ruleClass.ruleType().name())) { - throw new RuntimeException("No system rule updates allowed in user rule update."); + .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { + log.warn("System rule in user rule update."); + //throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRulesAndCheck(ruleFileBluePrintExisting); + removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); ruleFileBluePrintExisting.getRuleClasses() .stream() .flatMap(ruleClass -> ruleClass.ruleUnits() @@ -89,7 +81,6 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - log.info("finished merging user rules update with system rules"); RuleMergingResult mergingResult = RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) @@ -100,69 +91,17 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRulesAndCheck(RuleFileBluePrint ruleFileBluePrint) { - - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); - removeAllRulesExceptSystemRules(ruleFileBluePrint); - ruleFileBluePrint.getRuleClasses() - .forEach(ruleClass -> { - if (!systemRuleNames.contains(ruleClass.ruleType().name())) { - throw new RuntimeException("there was an error removing all rules except system rules"); - } - }); - } - - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - List rules = new ArrayList(); - Set systemRuleNames = systemRules.stream() - .map(RuleType::name) - .collect(Collectors.toSet()); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); - ruleFileBluePrint.getRuleClasses() - .stream() - .filter(ruleClass -> !systemRuleNames.contains(ruleClass.ruleType().name())) - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .map(ruleUnit -> new RuleIdentifier(ruleClass.ruleType(), ruleUnit.unit(), null))) - .forEach(rule -> rules.add(rule)); - rules.forEach(ruleIdentifier -> { - ruleFileBluePrint.removeRule(ruleIdentifier); - }); - } - - - private void removeSystemRulesAndCheckIfAnyRemain(RuleFileBluePrint ruleFileBluePrint) { - - removeSystemRules(ruleFileBluePrint); - for (RuleType systemRule : systemRules) { - List remainingSystemRules = new ArrayList(); - ruleFileBluePrint.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(remainingSystemRules::add)); - if (!remainingSystemRules.isEmpty()) { - throw new RuntimeException("There was an error removing the system rules from the file"); - } - } } private void removeSystemRules(RuleFileBluePrint ruleFileBluePrintExisting) { - for (RuleType systemRule : systemRules) { - List rules = new ArrayList(); - ruleFileBluePrintExisting.findRuleClassByType(systemRule) - .ifPresent(ruleClass -> ruleClass.ruleUnits() - .stream() - .forEach(rules::add)); - rules.forEach(ruleUnit -> { - ruleFileBluePrintExisting.removeRule(new RuleIdentifier(systemRule, ruleUnit.unit(), null)); - }); - ruleFileBluePrintExisting.removeRule(RuleIdentifier.fromRuleType(systemRule)); + for (RuleIdentifier systemRule : systemRules) { + ruleFileBluePrintExisting.removeRuleClassByRuleIdentifier(systemRule); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 616cabde..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -35,33 +35,6 @@ public class RuleCompilationResultParser { "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - public RuleCompilationResult buildBluePrintFromUserRulesString(String userRulesString) { - - DroolsValidation customDroolsValidation = DroolsValidation.builder().build(); - List allRules = new LinkedList<>(); - List allQueries = new LinkedList<>(); - PackageDescr packageDescr = new PackageDescr(); - String imports = ""; - String globals = ""; - List ruleClasses = buildRuleClasses(allRules); - return new RuleCompilationResult(imports.trim(), - packageDescr.getImports() - .stream() - .findFirst() - .map(ImportDescr::getLine) - .orElse(0), - globals.trim(), - packageDescr.getGlobals() - .stream() - .findFirst() - .map(GlobalDescr::getLine) - .orElse(0), - allQueries, - ruleClasses, - customDroolsValidation); - } - - @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { @@ -116,60 +89,7 @@ public class RuleCompilationResultParser { @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString) { - return buildRuleCompilationResultFromRuleString(ruleString,false); - } - - - @SneakyThrows - public String buildRulesStringFromBluePrint(RuleCompilationResult bluePrint) { - - StringBuilder ruleStringBuilder = new StringBuilder(); - - // Append imports - ruleStringBuilder.append(bluePrint.getImports()).append("\n\n"); - - // Append globals - if (!bluePrint.getGlobals().isEmpty()) { - ruleStringBuilder.append(bluePrint.getGlobals()).append("\n\n"); - } - - // Append queries - for (BasicQuery query : bluePrint.getQueries()) { - ruleStringBuilder.append(buildQueryString(query)).append("\n\n"); - } - - // Append rules - for (RuleClass ruleClass : bluePrint.getRuleClasses()) { - ruleStringBuilder.append(buildRuleString(ruleClass)).append("\n\n"); - } - - // Return the final rule string - return ruleStringBuilder.toString().trim(); - } - - - private String buildQueryString(BasicQuery query) { - - return "query \"" + query.getName() + "\"\n" + query.getCode() + "\n" + "end"; - } - - - private String buildRuleString(RuleClass ruleClass) { - - StringBuilder ruleBuilder = new StringBuilder(); - - // Use RuleType to distinguish between different rule types, if needed - ruleBuilder.append("rule \"").append(ruleClass.ruleType().name()).append("\"\n"); - - for (RuleUnit ruleUnit : ruleClass.ruleUnits()) { - for (BasicRule rule : ruleUnit.rules()) { - // Assuming BasicRule has a method to retrieve the condition as a string - ruleBuilder.append(rule.getCode()).append("\n"); - } - } - ruleBuilder.append("end"); - - return ruleBuilder.toString(); + return buildRuleCompilationResultFromRuleString(ruleString, false); } @@ -216,7 +136,6 @@ public class RuleCompilationResultParser { private List buildRuleClasses(List allRules) { - //todo: comments List ruleTypeOrder = allRules.stream() .map(BasicRule::getIdentifier) .map(RuleIdentifier::type) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 1996af83..181e108a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,149 +1,128 @@ package com.iqser.red.service.redaction.v1.server; +import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; import static org.mockito.Mockito.when; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileFactory; import com.iqser.red.service.redaction.v1.server.rulesmanagement.factory.RuleFileParser; import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.models.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.rulesmanagement.utils.RuleFileIO; import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; -import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; -import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(RuleBuilderTest.RuleBuilderTestConfiguration.class) -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class RuleBuilderTest extends AbstractRedactionIntegrationTest { +import org.junit.jupiter.api.Assertions; +import lombok.SneakyThrows; - private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); - private static final String USER_RULES = loadFromClassPath("rulesmanagement/userrules/user_rule_update.drl"); - private final List systemRules = new ArrayList<>(Arrays.asList("AI", "MAN", "X", "DICT", "FA", "LDS")); - - @Autowired - RuleBuilderService ruleBuilderService; - - @Configuration - @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) - @Import(LayoutParsingServiceProcessorConfiguration.class) - @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) - public static class RuleBuilderTestConfiguration { - - @Bean - @Primary - public StorageService inmemoryStorage() { - - return new FileSystemBackedStorageService(ObjectMapperFactory.create()); - } - - } +class RuleBuilderTest { + private RuleBuilderService ruleBuilderService; + private final List systemRules = new ArrayList<>(Arrays.asList( + RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*") + )); @BeforeEach - public void stubClients() { - + void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); - - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); - - loadDictionaryForTest(); - loadTypeForTest(); - loadNerForTest(); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); - - when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); - - mockDictionaryCalls(null); - - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } - @Test - public void removeSystemRulesTest() { + @SneakyThrows + void removeSystemRulesTest() throws IOException { + URL url = getClass().getClassLoader().getResource("drools"); + if (url == null) { + throw new RuntimeException("Drools directory not found in resources"); + } - String cleanedRulesWithImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, false); - String cleanedRulesWithoutImports = this.ruleBuilderService.cleanRuleFileOfSystemRules(RULES, true); - RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + Path path = Paths.get(url.toURI()); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); - Assert.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); - - Assert.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); - Assert.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); + RuleFileIO.streamAllRuleFilesInDirectory(path) + .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } + private void testThatSystemRulesAreRemoved(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + true + ); - @Test - public void mergeUserRulesUpdateTest() { + RuleCompilationResult ruleCompilationResultWithImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = + RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); - RuleMergingResult mergingResult = this.ruleBuilderService.mergeUserRulesAndSystemRules(RULES, USER_RULES); - String mergedRules = mergingResult.getMergedRules(); - RuleFileBluePrint ruleFileBluePrintUserRulesUpdate = RuleFileParser.buildBluePrintFromRulesString(mergedRules, true); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(mergedRules); - ruleFileBluePrintUserRulesUpdate.getRuleClasses() - .forEach(ruleClass -> { - Assert.assertTrue(ruleFileBluePrint.getRuleClasses().contains(ruleClass)); - }); + Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithImports)); + Assertions.assertTrue(checkIfImportsDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithoutImports)); + Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + false + ); + + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( + RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports + ); + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + + Assertions.assertEquals( + RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), + cleanedRuleFileMergedWithSystemRules.getMergedRules() + ); + } private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getRuleClasses() - .stream() - .filter(ruleClass -> this.systemRules.contains(ruleClass.ruleType().name())) - .collect(Collectors.toList()).size() == 0; + .stream() + .map(RuleClass::ruleType) + .noneMatch(ruleType -> systemRules.stream() + .map(RuleIdentifier::type) + .anyMatch(type -> ruleType.name().equals(type.name()))); } - private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getImports().isEmpty(); } - private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { - return ruleCompilationResult.getQueries().isEmpty(); } - -} +} \ No newline at end of file -- 2.47.2 From 67b2d120ce97723e87976fbf05984d78b568891f Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 065/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From f568074983bc9cd43515267abe696d82880984bd Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 066/108] RED-9472: seperation of system rules pmd fix --- .../v1/server/service/RuleBuilderService.java | 20 +++++++++---------- .../drools/RuleCompilationResultParser.java | 8 ++------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index d20e8337..2c9c9f7e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*"))); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -58,7 +58,7 @@ public class RuleBuilderService { if (systemRules.stream() .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { log.warn("System rule in user rule update."); - //throw new RuntimeException("No system rule updates allowed in user rule update."); + throw new RuntimeException("No system rule updates allowed in user rule update."); } }); removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); @@ -81,19 +81,17 @@ public class RuleBuilderService { mergedRuleFileBlueprint.getDeclarations() .stream()) .toList()); - RuleMergingResult mergingResult = RuleMergingResult.builder() + return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) .build(); - - return mergingResult; } private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From fa55aca9930b8a86da89e0d6549675077474181a Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:26:32 +0100 Subject: [PATCH 067/108] RED-9472: seperation of system rules pmd fix --- .../redaction/v1/server/RuleBuilderTest.java | 83 ++++++++----------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java index 181e108a..31d61e1e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RuleBuilderTest.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server; -import static com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest.TEST_DOSSIER_TEMPLATE_ID; -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URL; import java.nio.file.Path; @@ -13,8 +8,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import com.iqser.red.service.redaction.v1.server.model.RuleMergingResult; import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; import com.iqser.red.service.redaction.v1.server.model.drools.RuleCompilationResult; @@ -27,35 +24,32 @@ import com.iqser.red.service.redaction.v1.server.service.RuleBuilderService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.knecon.fforesight.tenantcommons.TenantContext; -import org.junit.jupiter.api.Assertions; import lombok.SneakyThrows; class RuleBuilderTest { private RuleBuilderService ruleBuilderService; - private final List systemRules = new ArrayList<>(Arrays.asList( - RuleIdentifier.fromName("AI.*.*"), - RuleIdentifier.fromName("MAN.*.*"), - RuleIdentifier.fromName("X.*.*"), - RuleIdentifier.fromName("DICT.*.*"), - RuleIdentifier.fromName("FA.*.*"), - RuleIdentifier.fromName("LDS.*.*") - )); + private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI.*.*"), + RuleIdentifier.fromName("MAN.*.*"), + RuleIdentifier.fromName("X.*.*"), + RuleIdentifier.fromName("DICT.*.*"), + RuleIdentifier.fromName("FA.*.*"), + RuleIdentifier.fromName("LDS.*.*"))); + @BeforeEach void setUp() { + ruleBuilderService = new RuleBuilderService(); TenantContext.setTenantId("redaction"); } + @Test @SneakyThrows void removeSystemRulesTest() throws IOException { - URL url = getClass().getClassLoader().getResource("drools"); - if (url == null) { - throw new RuntimeException("Drools directory not found in resources"); - } + URL url = Thread.currentThread().getContextClassLoader().getResource("drools"); Path path = Paths.get(url.toURI()); RuleFileIO.streamAllRuleFilesInDirectory(path) .forEach(file -> testThatSystemRulesAreRemoved(file.toPath())); @@ -64,20 +58,14 @@ class RuleBuilderTest { .forEach(file -> testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(file.toPath())); } - private void testThatSystemRulesAreRemoved(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - true - ); - RuleCompilationResult ruleCompilationResultWithImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); - RuleCompilationResult ruleCompilationResultWithoutImports = - RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); + private void testThatSystemRulesAreRemoved(Path path) { + + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); + String cleanedRulesWithoutImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), true); + + RuleCompilationResult ruleCompilationResultWithImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithImports, false); + RuleCompilationResult ruleCompilationResultWithoutImports = RuleCompilationResultParser.buildRuleCompilationResultFromRuleString(cleanedRulesWithoutImports, true); Assertions.assertFalse(checkIfImportsDontExist(ruleCompilationResultWithImports)); Assertions.assertTrue(checkIfSystemRulesDontExist(ruleCompilationResultWithImports)); @@ -88,28 +76,22 @@ class RuleBuilderTest { Assertions.assertTrue(checkIfQueriesDontExist(ruleCompilationResultWithoutImports)); } + private void testRuleFilesRemainSameAfterSeperationAndReaddingSystemrules(Path path) { - String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - false - ); - RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules( - RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), - cleanedRulesWithImports - ); + String cleanedRulesWithImports = ruleBuilderService.cleanRuleFileOfSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), false); - RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString( - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleMergingResult cleanedRuleFileMergedWithSystemRules = ruleBuilderService.mergeUserRulesAndSystemRules(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()), + cleanedRulesWithImports); - Assertions.assertEquals( - RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), - cleanedRuleFileMergedWithSystemRules.getMergedRules() - ); + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(cleanedRuleFileMergedWithSystemRules.getMergedRules()); + + Assertions.assertEquals(RuleFileFactory.buildRuleString(ruleFileBluePrint, false, false), cleanedRuleFileMergedWithSystemRules.getMergedRules()); } + private boolean checkIfSystemRulesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getRuleClasses() .stream() .map(RuleClass::ruleType) @@ -118,11 +100,16 @@ class RuleBuilderTest { .anyMatch(type -> ruleType.name().equals(type.name()))); } + private boolean checkIfImportsDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getImports().isEmpty(); } + private boolean checkIfQueriesDontExist(RuleCompilationResult ruleCompilationResult) { + return ruleCompilationResult.getQueries().isEmpty(); } + } \ No newline at end of file -- 2.47.2 From 1126d07540df8366f2b0b61465cec89f8833dd5b Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 15 Nov 2024 09:45:56 +0100 Subject: [PATCH 068/108] RED-9472: seperation of system rules review changes --- .../models/RuleFileBluePrint.java | 2 +- .../v1/server/service/RuleBuilderService.java | 46 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index ca185b63..533492e1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -180,7 +180,7 @@ public class RuleFileBluePrint { public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set identifiers) { - RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), new LinkedList<>(), new LinkedList<>()); + RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports, globals, queries, new LinkedList<>(), declarations, functions); ruleClasses.stream() .flatMap(ruleClass -> ruleClass.ruleUnits() .stream()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 2c9c9f7e..c346429c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -1,10 +1,10 @@ package com.iqser.red.service.redaction.v1.server.service; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -22,12 +22,12 @@ import lombok.extern.slf4j.Slf4j; @Service public class RuleBuilderService { - private final List systemRules = new ArrayList<>(Arrays.asList(RuleIdentifier.fromName("AI"), - RuleIdentifier.fromName("MAN"), - RuleIdentifier.fromName("X"), - RuleIdentifier.fromName("DICT"), - RuleIdentifier.fromName("FA"), - RuleIdentifier.fromName("LDS"))); + private final HashSet systemRules = new HashSet<>(Set.of(RuleIdentifier.fromName("AI"), + RuleIdentifier.fromName("MAN"), + RuleIdentifier.fromName("X"), + RuleIdentifier.fromName("DICT"), + RuleIdentifier.fromName("FA"), + RuleIdentifier.fromName("LDS"))); public RuleBuilderModel getRuleBuilderModel() { @@ -52,7 +52,7 @@ public class RuleBuilderService { public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, true); + RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() @@ -61,25 +61,30 @@ public class RuleBuilderService { throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - ruleFileBluePrintExisting.getRuleClasses() - .stream() - .flatMap(ruleClass -> ruleClass.ruleUnits() - .stream() - .flatMap(ruleUnit -> ruleUnit.rules() - .stream())) + ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); + ruleFileBluePrintExisting.getAllRules() .forEach(mergedRuleFileBlueprint::addRule); - mergedRuleFileBlueprint.setImports(ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()); - mergedRuleFileBlueprint.setGlobals(ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()); + Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setImports(uniqueImports.stream() + .collect(Collectors.joining(""))); + Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); + mergedRuleFileBlueprint.setGlobals(uniqueGlobals.stream() + .collect(Collectors.joining(""))); mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() .stream(), mergedRuleFileBlueprint.getFunctions() .stream()) + .distinct() .toList()); mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() .stream(), mergedRuleFileBlueprint.getDeclarations() .stream()) + .distinct() .toList()); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) @@ -89,10 +94,9 @@ public class RuleBuilderService { } - private void removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { - - ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); + private RuleFileBluePrint removeAllRulesExceptSystemRules(RuleFileBluePrint ruleFileBluePrint) { + return ruleFileBluePrint.buildFilteredBluePrintByRuleIdentifiers(new HashSet(systemRules)); } -- 2.47.2 From e08e43e6839f792d69aab189009e44f7030b40eb Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 069/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 94fde34ff086fa02465484418b805caa6d8238aa Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 070/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From d178f4728cd4b531626a7d98c7af91814aeea5f6 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 09:04:46 +0100 Subject: [PATCH 071/108] RED-9472: seperation of system rules added document dependency --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index f3d723ae..d2524cb6 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -38,7 +38,7 @@ configurations.all { } dependencies { - + implementation(project(":document")) implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") -- 2.47.2 From 33d921dacd08ebcdf6aea303e8a793f9880cd54e Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 13:02:03 +0100 Subject: [PATCH 072/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index d2524cb6..7a308094 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -45,6 +45,7 @@ dependencies { { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } + implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}") implementation("com.iqser.red.commons:spring-commons:6.2.0") @@ -62,6 +63,7 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") implementation("org.ahocorasick:ahocorasick:0.6.3") + implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") -- 2.47.2 From dcff38a247ff17550f9f336779f20d8016fa8f39 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Sep 2024 10:24:59 +0200 Subject: [PATCH 073/108] RED-9472: seperation of system rules added endpoints for seperating and readding system rules working on service RED-9472: seperation of system rules refactored endpoints. now removing system rules on download merging user rules with systemrules on upload RED-9472: seperation of system rules RED-9472: seperation of system rules RED-9472: seperation of system rules refactored two have a rulefileblueprint and a rulefileblueprint for validationresult changed the flow of merging and seperating system and user rules added functions and declarations to rule file parsing RED-9472: seperation of system rules RED-9472: seperation of system rules introduced tests working on fixing bug with merging rules RED-9472: seperation of system rules introduced tests RED-9472: seperation of system rules fixed circular depdency RED-9472: seperation of system rules updated persistence service version RED-9472: seperation of system rules refactored rules refactored pcackages RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed failing tests RED-9472: seperation of system rules fixed some sonarqube errors RED-9472: seperation of system rules fixed pmd error RED-9472: seperation of system rules added deploy stage to gitlab pipeline RED-9472: seperation of system rules RED-9472: seperation of system rules fixed a mistake RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse RED-9472: seperation of system rules updated controller to return RulesUploadResponse including the droolsvalidationresponse --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../redaction-service-server-v1/build.gradle.kts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index dc942464..ccff795e 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.612.0-RED10072.1" +val persistenceServiceVersion = "2.585.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 7a308094..707d8690 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,8 +16,8 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.612.0-RED10072.1" -val llmServiceVersion = "1.20.0-RED10072.2" +val persistenceServiceVersion = "2.585.0-RED9472.1" +val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" -- 2.47.2 From 4c3d0ce10eccc857b909f3819fb33b240d4a1446 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 074/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From d27e12f9508fc9b38854b8c11b25a46db5cc8c31 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 075/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From b1f2c9684b7df6444db221ecc6424785faa10630 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 076/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 783f14600ca78e9ff791d201b560f741c472c94f Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 077/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 37ee7f0f78b7d48621e98c3635165d6957735dd3 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 18 Nov 2024 13:02:03 +0100 Subject: [PATCH 078/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 707d8690..67c38388 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.585.0-RED9472.1" +val persistenceServiceVersion = "2.627.0-RED9472.2" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" -- 2.47.2 From c1c0f4cfd76a51a59f7e9a145e4f74be1888e58e Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 08:52:32 +0100 Subject: [PATCH 079/108] RED-9472: seperation of system rules added updated persistence service version --- .../redaction-service-server-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 67c38388..2714afe8 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.627.0-RED9472.2" +val persistenceServiceVersion = "2.628.0-RED9472.0" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" -- 2.47.2 From 19ee5a532288fe3d4a9da4b7daeaa33cdcbc16ee Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 09:55:55 +0100 Subject: [PATCH 080/108] RED-9472: seperation of system rules added updated persistence service version --- redaction-service-v1/redaction-service-api-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index ccff795e..811fa166 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.585.0-RED9472.0" +val persistenceServiceVersion = "2.628.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") -- 2.47.2 From ff05da06d9c804375356c8dbd82c67cd87dde44b Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 10:04:51 +0100 Subject: [PATCH 081/108] RED-9472: seperation of system rules added updated service version for fixing checkstyle errors --- .../redaction-service-server-v1/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 2714afe8..36f2fbb9 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -53,7 +53,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.50.0") - implementation("com.knecon.fforesight:tenant-commons:0.30.0") + implementation("com.knecon.fforesight:tenant-commons:0.31.0") implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") } @@ -64,6 +64,8 @@ dependencies { implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") implementation("org.ahocorasick:ahocorasick:0.6.3") implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") + implementation("com.github.roklenarcic:aho-corasick:1.2") + implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") -- 2.47.2 From 917e7117eb3c7a37fe0509295fbd06579cddc3bf Mon Sep 17 00:00:00 2001 From: yhampe Date: Tue, 19 Nov 2024 10:18:10 +0100 Subject: [PATCH 082/108] RED-9472: seperation of system rules added updated service version for fixing checkstyle errors --- .../v1/server/service/drools/DroolsValidationService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java index e27361be..48467c63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java @@ -105,7 +105,6 @@ public class DroolsValidationService { .line(ruleCompilationResult.getGlobalsLine()) .column(0) .build(); - customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint)); } -- 2.47.2 From e27498ebd3366fcb8f89396c34399cbac0c79707 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Dec 2024 13:35:51 +0100 Subject: [PATCH 083/108] RED-9472: seperation of system rules --- .../service/redaction/v1/server/service/RuleBuilderService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index c346429c..54670ba4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -86,6 +86,7 @@ public class RuleBuilderService { .stream()) .distinct() .toList()); + log.info("merged rules: {}", RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) -- 2.47.2 From bc7943979c5cdc4124328ab9239b3e4f6e957ea7 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 084/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From a036bed9c356d73441af827a464566a93cd9a3e2 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 085/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 0ccadb43fde9b8ff98af26e4fa01e67352ed0fcb Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 086/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From e071b1c633d5218f34fdd8877568ed7b53b183bb Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 087/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From faea7bcd76f4aa74c94856288f7c85fc50718253 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 088/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 7b095ac70672938551d3b3b3adf69014df467be4 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 089/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From f75aa7389d1c2e412a91f279cfa38026470fe0db Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 11:38:37 +0100 Subject: [PATCH 090/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 65ad8cad..8e8ae2e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,8 +32,12 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); - + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + + "\\s*\\(\\s*" + + "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + + "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + + "\\s*\\)" + ); @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From db1f5aae0f15568de5906a07c375754e73e3a48c Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 13 Nov 2024 12:01:07 +0100 Subject: [PATCH 091/108] RED-9472: seperation of system rules pmd fix --- .../service/drools/RuleCompilationResultParser.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java index 8e8ae2e3..65ad8cad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleCompilationResultParser.java @@ -32,12 +32,8 @@ import lombok.experimental.UtilityClass; public class RuleCompilationResultParser { private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( - "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)" + - "\\s*\\(\\s*" + - "\"([a-zA-Z0-9]+\\.\\d+\\.\\d+)\"" + - "\\s*,\\s*[^,\\)]*(?:\\s*,\\s*[^,\\)]*)*" + - "\\s*\\)" - ); + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)"); + @SneakyThrows public RuleCompilationResult buildRuleCompilationResultFromRuleString(String ruleString, boolean removedImports) { -- 2.47.2 From 3e82b997f6482eae69d22ed95e97eeb9a709282e Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Dec 2024 15:00:01 +0100 Subject: [PATCH 092/108] RED-9472: seperation of system rules --- redaction-service-v1/redaction-service-api-v1/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 811fa166..97d4351d 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.628.0-RED9472.0" +val persistenceServiceVersion = "2.649.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") -- 2.47.2 From 7363493d1c1495f7ab66addc029bbe2abfc919ea Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 12 Dec 2024 15:04:47 +0100 Subject: [PATCH 093/108] RED-9472: seperation of system rules --- .../files/management/services/DroolsValidationServiceTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index 0bd9ecec..a566c65f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -27,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidation import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleCompilationResultParser; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; +import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService; import lombok.SneakyThrows; -- 2.47.2 From bdb29a0e359918584f5312b41ca99e79d928bd9c Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 11:52:36 +0100 Subject: [PATCH 094/108] RED-9472: seperation of system rules now adding user update rules to existing blueprint and returning this --- .../controller/RuleBuilderController.java | 2 +- .../v1/server/service/RuleBuilderService.java | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index aa215fc7..8c452b07 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -93,13 +93,13 @@ public class RuleBuilderController implements RuleBuilderResource { rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.UNPROCESSABLE_ENTITY); } else { - // rulesUploadResponse.setRules(mergingResult.getMergedRules()); rulesUploadResponse.setDroolsValidationResponse(droolsValidationResponse); } } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } + log.info("result after validation: {}",rulesUpdateRequest); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 54670ba4..e4d023a2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -53,6 +53,7 @@ public class RuleBuilderService { RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); + mergedRuleFileBlueprint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() @@ -62,25 +63,25 @@ public class RuleBuilderService { } }); ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - ruleFileBluePrintExisting.getAllRules() - .forEach(mergedRuleFileBlueprint::addRule); + mergedRuleFileBlueprint.getAllRules() + .forEach(ruleFileBluePrintExisting::addRule); Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); - mergedRuleFileBlueprint.setImports(uniqueImports.stream() + ruleFileBluePrintExisting.setImports(uniqueImports.stream() .collect(Collectors.joining(""))); Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); - mergedRuleFileBlueprint.setGlobals(uniqueGlobals.stream() + ruleFileBluePrintExisting.setGlobals(uniqueGlobals.stream() .collect(Collectors.joining(""))); - mergedRuleFileBlueprint.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() + ruleFileBluePrintExisting.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() .stream(), mergedRuleFileBlueprint.getFunctions() .stream()) .distinct() .toList()); - mergedRuleFileBlueprint.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() + ruleFileBluePrintExisting.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() .stream(), mergedRuleFileBlueprint.getDeclarations() .stream()) @@ -88,9 +89,9 @@ public class RuleBuilderService { .toList()); log.info("merged rules: {}", RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)); return RuleMergingResult.builder() - .mergedRules(RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)) - .addedGlobalsOffset(ruleFileBluePrintExisting.getGlobals().length()) - .addedImportsOffset(ruleFileBluePrintExisting.getImports().length()) + .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBlueprint, false, false)) + .addedGlobalsOffset(mergedRuleFileBlueprint.getGlobals().length()) + .addedImportsOffset(mergedRuleFileBlueprint.getImports().length()) .build(); } -- 2.47.2 From d2994f9e14a25eea1c025de10f93272eabb02d29 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 13:19:16 +0100 Subject: [PATCH 095/108] RED-9472: seperation of system rules adding logs for bug investigation --- .../controller/RuleBuilderController.java | 3 ++- .../v1/server/service/RuleBuilderService.java | 25 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 8c452b07..7d0957b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -63,6 +63,7 @@ public class RuleBuilderController implements RuleBuilderResource { } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); + log.info("result of validation: " + droolsValidation); droolsValidationResponse = DroolsValidationResponse.builder() .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() .stream() @@ -99,7 +100,7 @@ public class RuleBuilderController implements RuleBuilderResource { } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } - log.info("result after validation: {}",rulesUpdateRequest); + log.info("result after validation: {}", rulesUpdateRequest); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index e4d023a2..d3a7352e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -69,25 +69,24 @@ public class RuleBuilderService { .map(String::trim) .collect(Collectors.toSet()); ruleFileBluePrintExisting.setImports(uniqueImports.stream() - .collect(Collectors.joining(""))); + .collect(Collectors.joining(""))); Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) .map(String::trim) .collect(Collectors.toSet()); ruleFileBluePrintExisting.setGlobals(uniqueGlobals.stream() - .collect(Collectors.joining(""))); + .collect(Collectors.joining(""))); ruleFileBluePrintExisting.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() - .stream(), - mergedRuleFileBlueprint.getFunctions() - .stream()) - .distinct() - .toList()); + .stream(), + mergedRuleFileBlueprint.getFunctions() + .stream()) + .distinct() + .toList()); ruleFileBluePrintExisting.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() - .stream(), - mergedRuleFileBlueprint.getDeclarations() - .stream()) - .distinct() - .toList()); - log.info("merged rules: {}", RuleFileFactory.buildRuleString(ruleFileBluePrintExisting, false, false)); + .stream(), + mergedRuleFileBlueprint.getDeclarations() + .stream()) + .distinct() + .toList()); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBlueprint, false, false)) .addedGlobalsOffset(mergedRuleFileBlueprint.getGlobals().length()) -- 2.47.2 From 77a76f8b96f26f29500e870213b0caacb0c61464 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 13:42:08 +0100 Subject: [PATCH 096/108] RED-9472: seperation of system rules adding logs for bug investigation --- .../redaction/v1/server/controller/RuleBuilderController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 7d0957b9..30aeadc6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -62,6 +62,7 @@ public class RuleBuilderController implements RuleBuilderResource { throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { + log.info("result"+mergingResult.getMergedRules()); var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); log.info("result of validation: " + droolsValidation); droolsValidationResponse = DroolsValidationResponse.builder() -- 2.47.2 From 1a16dd466956c577c1ca7e1087f13b8f8ad2c0a8 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 14:46:20 +0100 Subject: [PATCH 097/108] RED-9472: seperation of system rules added a method for merging imports to preserve the package description --- .../controller/RuleBuilderController.java | 1 - .../models/RuleFileBluePrint.java | 58 +++++++++++++++++ .../v1/server/service/RuleBuilderService.java | 63 ++++++++++--------- 3 files changed, 92 insertions(+), 30 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 30aeadc6..7d0957b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -62,7 +62,6 @@ public class RuleBuilderController implements RuleBuilderResource { throw new AssertionError("There was an error when merging the user rule update into the rule file"); } try { - log.info("result"+mergingResult.getMergedRules()); var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); log.info("result of validation: " + droolsValidation); droolsValidationResponse = DroolsValidationResponse.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 533492e1..196e7f85 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.rulesmanagement.models; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; @@ -34,6 +35,9 @@ public class RuleFileBluePrint { List declarations; List functions; + private static final String IMPORT_PREFIX = "import "; + private static final String PACKAGE_PREFIX = "package "; + public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -54,6 +58,60 @@ public class RuleFileBluePrint { } + public void addImport(String importStatement) { + + if (importStatement == null || importStatement.trim().isEmpty()) { + return; + } + + String cleanImport = importStatement.trim(); + if (!cleanImport.startsWith(IMPORT_PREFIX)) { + cleanImport = IMPORT_PREFIX + cleanImport; + } + + if (imports == null) { + imports = PACKAGE_PREFIX + "drools\n\n"; + } + + if (!imports.contains(cleanImport)) { + imports = imports.trim() + "\n" + cleanImport; + } + } + + + public void mergeImports(RuleFileBluePrint other) { + + if (other == null || other.getImports() == null) { + return; + } + + addImports(other.getImports()); + } + + + public String getImports() { + + if (imports == null || imports.trim().isEmpty()) { + return PACKAGE_PREFIX + "drools\n\n"; + } + return imports; + } + + + public void addImports(String importStatements) { + + if (importStatements == null || importStatements.trim().isEmpty()) { + return; + } + + Arrays.stream(importStatements.split("\n")) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .filter(line -> !line.startsWith(PACKAGE_PREFIX)) + .forEach(this::addImport); + } + + public boolean removeRuleClassByRuleIdentifier(RuleIdentifier ruleIdentifier) { AtomicBoolean wasRemoved = new AtomicBoolean(false); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index d3a7352e..5c215c2c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -51,10 +51,10 @@ public class RuleBuilderService { public RuleMergingResult mergeUserRulesAndSystemRules(String existingRules, String userUpdatedRules) { - RuleFileBluePrint ruleFileBluePrintExisting = RuleFileParser.buildBluePrintFromRulesString(existingRules); - RuleFileBluePrint mergedRuleFileBlueprint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); + RuleFileBluePrint mergedRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(existingRules); + RuleFileBluePrint userRulesBluePrint = RuleFileParser.buildBluePrintFromRulesString(userUpdatedRules, false); - mergedRuleFileBlueprint.getRuleClasses() + userRulesBluePrint.getRuleClasses() .forEach(ruleClass -> { if (systemRules.stream() .toList().contains(RuleIdentifier.fromName(ruleClass.ruleType().name()))) { @@ -62,35 +62,40 @@ public class RuleBuilderService { throw new RuntimeException("No system rule updates allowed in user rule update."); } }); - ruleFileBluePrintExisting = removeAllRulesExceptSystemRules(ruleFileBluePrintExisting); - mergedRuleFileBlueprint.getAllRules() - .forEach(ruleFileBluePrintExisting::addRule); - Set uniqueImports = Arrays.stream((ruleFileBluePrintExisting.getImports() + mergedRuleFileBlueprint.getImports()).replaceAll("\n", "").split("import")) + + mergedRuleFileBluePrint = removeAllRulesExceptSystemRules(mergedRuleFileBluePrint); + userRulesBluePrint.getAllRules() + .forEach(mergedRuleFileBluePrint::addRule); + + mergedRuleFileBluePrint.mergeImports(userRulesBluePrint); + + Set uniqueGlobals = Arrays.stream((mergedRuleFileBluePrint.getGlobals() + userRulesBluePrint.getGlobals()).replaceAll("\n", "").split("global")) .map(String::trim) + .filter(s -> !s.isEmpty()) .collect(Collectors.toSet()); - ruleFileBluePrintExisting.setImports(uniqueImports.stream() - .collect(Collectors.joining(""))); - Set uniqueGlobals = Arrays.stream((ruleFileBluePrintExisting.getGlobals() + mergedRuleFileBlueprint.getGlobals()).replaceAll("\n", "").split("import")) - .map(String::trim) - .collect(Collectors.toSet()); - ruleFileBluePrintExisting.setGlobals(uniqueGlobals.stream() - .collect(Collectors.joining(""))); - ruleFileBluePrintExisting.setFunctions(Stream.concat(ruleFileBluePrintExisting.getFunctions() - .stream(), - mergedRuleFileBlueprint.getFunctions() - .stream()) - .distinct() - .toList()); - ruleFileBluePrintExisting.setDeclarations(Stream.concat(ruleFileBluePrintExisting.getDeclarations() - .stream(), - mergedRuleFileBlueprint.getDeclarations() - .stream()) - .distinct() - .toList()); + + mergedRuleFileBluePrint.setGlobals(uniqueGlobals.stream() + .map(global -> "global " + global) + .collect(Collectors.joining("\n"))); + + mergedRuleFileBluePrint.setFunctions(Stream.concat(mergedRuleFileBluePrint.getFunctions() + .stream(), + userRulesBluePrint.getFunctions() + .stream()) + .distinct() + .toList()); + + mergedRuleFileBluePrint.setDeclarations(Stream.concat(mergedRuleFileBluePrint.getDeclarations() + .stream(), + userRulesBluePrint.getDeclarations() + .stream()) + .distinct() + .toList()); + return RuleMergingResult.builder() - .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBlueprint, false, false)) - .addedGlobalsOffset(mergedRuleFileBlueprint.getGlobals().length()) - .addedImportsOffset(mergedRuleFileBlueprint.getImports().length()) + .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBluePrint, false, false)) + .addedGlobalsOffset(userRulesBluePrint.getGlobals().length()) + .addedImportsOffset(userRulesBluePrint.getImports().length()) .build(); } -- 2.47.2 From cdc0830700061b47f89ebf608d5e09f91359ba25 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 15:11:38 +0100 Subject: [PATCH 098/108] RED-9472: seperation of system rules added methods for mergin: functions, globals, declarations to RuleFileBluePrint --- .../models/RuleFileBluePrint.java | 61 +++++++++++++++++++ .../v1/server/service/RuleBuilderService.java | 26 +------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 196e7f85..303430c3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -11,6 +12,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -37,6 +39,7 @@ public class RuleFileBluePrint { private static final String IMPORT_PREFIX = "import "; private static final String PACKAGE_PREFIX = "package "; + private static final String GLOBAL_PREFIX = "global "; public boolean removeRule(RuleIdentifier ruleIdentifier) { @@ -58,6 +61,64 @@ public class RuleFileBluePrint { } + public void mergeGlobals(RuleFileBluePrint other) { + + if (other == null || other.getGlobals() == null) { + return; + } + + Set existingGlobals = new LinkedHashSet<>(Arrays.stream(this.globals.split("\n")) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .toList()); + + Set newGlobals = Arrays.stream(other.getGlobals().split("\n")) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + StringBuilder globalsBuilder = new StringBuilder(); + Stream.concat(existingGlobals.stream(), newGlobals.stream()) + .distinct() + .forEach(global -> { + if (!global.startsWith(GLOBAL_PREFIX)) { + global = GLOBAL_PREFIX + global; + } + globalsBuilder.append(global).append("\n"); + }); + + this.globals = globalsBuilder.toString().trim(); + } + + + public void mergeFunctions(RuleFileBluePrint other) { + + if (other == null || other.getFunctions() == null) { + return; + } + + this.functions = Stream.concat(this.functions.stream(), + other.getFunctions() + .stream()) + .distinct() + .toList(); + } + + + public void mergeDeclarations(RuleFileBluePrint other) { + + if (other == null || other.getDeclarations() == null) { + return; + } + + this.declarations = Stream.concat(this.declarations.stream(), + other.getDeclarations() + .stream()) + .distinct() + .toList(); + } + + public void addImport(String importStatement) { if (importStatement == null || importStatement.trim().isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java index 5c215c2c..fa1c91b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RuleBuilderService.java @@ -68,29 +68,9 @@ public class RuleBuilderService { .forEach(mergedRuleFileBluePrint::addRule); mergedRuleFileBluePrint.mergeImports(userRulesBluePrint); - - Set uniqueGlobals = Arrays.stream((mergedRuleFileBluePrint.getGlobals() + userRulesBluePrint.getGlobals()).replaceAll("\n", "").split("global")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toSet()); - - mergedRuleFileBluePrint.setGlobals(uniqueGlobals.stream() - .map(global -> "global " + global) - .collect(Collectors.joining("\n"))); - - mergedRuleFileBluePrint.setFunctions(Stream.concat(mergedRuleFileBluePrint.getFunctions() - .stream(), - userRulesBluePrint.getFunctions() - .stream()) - .distinct() - .toList()); - - mergedRuleFileBluePrint.setDeclarations(Stream.concat(mergedRuleFileBluePrint.getDeclarations() - .stream(), - userRulesBluePrint.getDeclarations() - .stream()) - .distinct() - .toList()); + mergedRuleFileBluePrint.mergeGlobals(userRulesBluePrint); + mergedRuleFileBluePrint.mergeFunctions(userRulesBluePrint); + mergedRuleFileBluePrint.mergeDeclarations(userRulesBluePrint); return RuleMergingResult.builder() .mergedRules(RuleFileFactory.buildRuleString(mergedRuleFileBluePrint, false, false)) -- 2.47.2 From dfae9e5d1db8a31a5a40660736d5bda60d0c1996 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 13 Dec 2024 15:27:41 +0100 Subject: [PATCH 099/108] RED-9472: seperation of system rules fixed checkstyle --- .../server/rulesmanagement/models/RuleFileBluePrint.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 303430c3..420cc936 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -75,17 +75,14 @@ public class RuleFileBluePrint { Set newGlobals = Arrays.stream(other.getGlobals().split("\n")) .map(String::trim) .filter(line -> !line.isEmpty()) + .map(line -> line.startsWith(GLOBAL_PREFIX) ? line : GLOBAL_PREFIX + line) .collect(Collectors.toCollection(LinkedHashSet::new)); + // Combine globals while maintaining order and format StringBuilder globalsBuilder = new StringBuilder(); Stream.concat(existingGlobals.stream(), newGlobals.stream()) .distinct() - .forEach(global -> { - if (!global.startsWith(GLOBAL_PREFIX)) { - global = GLOBAL_PREFIX + global; - } - globalsBuilder.append(global).append("\n"); - }); + .forEach(globalLine -> globalsBuilder.append(globalLine).append("\n")); this.globals = globalsBuilder.toString().trim(); } -- 2.47.2 From e55c4e5fdaef5d7a1197caf687bc89b92d7deb72 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 18 Dec 2024 09:49:07 +0100 Subject: [PATCH 100/108] RED-9472: seperation of system rules working on fixing bug --- .../redaction/v1/server/controller/RuleBuilderController.java | 3 +-- .../v1/server/rulesmanagement/models/RuleFileBluePrint.java | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 7d0957b9..a61423e9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -63,7 +63,6 @@ public class RuleBuilderController implements RuleBuilderResource { } try { var droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), mergingResult.getMergedRules())); - log.info("result of validation: " + droolsValidation); droolsValidationResponse = DroolsValidationResponse.builder() .syntaxErrorMessages(droolsValidation.getSyntaxErrorMessages() .stream() @@ -100,7 +99,7 @@ public class RuleBuilderController implements RuleBuilderResource { } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } - log.info("result after validation: {}", rulesUpdateRequest); + log.info("result after validation: {}", rulesUploadResponse); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java index 420cc936..d2aba3d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/rulesmanagement/models/RuleFileBluePrint.java @@ -78,7 +78,6 @@ public class RuleFileBluePrint { .map(line -> line.startsWith(GLOBAL_PREFIX) ? line : GLOBAL_PREFIX + line) .collect(Collectors.toCollection(LinkedHashSet::new)); - // Combine globals while maintaining order and format StringBuilder globalsBuilder = new StringBuilder(); Stream.concat(existingGlobals.stream(), newGlobals.stream()) .distinct() -- 2.47.2 From 04eb6cf3f8b586e8ddff41e7cbb4c1c6ea457e94 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 18 Dec 2024 10:47:59 +0100 Subject: [PATCH 101/108] RED-9472: seperation of system rules working on fixing bug --- redaction-service-v1/redaction-service-api-v1/build.gradle.kts | 2 +- .../redaction/v1/server/controller/RuleBuilderController.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 97d4351d..3c05f811 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.649.0-RED9472.0" +val persistenceServiceVersion = "2.651.0-RED9472.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index a61423e9..a54de98a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -99,7 +99,7 @@ public class RuleBuilderController implements RuleBuilderResource { } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } - log.info("result after validation: {}", rulesUploadResponse); + log.info("response: " + new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK)); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } -- 2.47.2 From 5ab994969a9394f6fac819f85e86e4abb6bf7e94 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 19 Dec 2024 13:44:58 +0100 Subject: [PATCH 102/108] RED-9742: fix document import --- ...er.red.service.java-conventions.gradle.kts | 12 +++++ .../document/build.gradle.kts | 5 +- .../build.gradle.kts | 53 ++++++++++--------- .../document/EntityCreationService.java | 4 +- 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts index 4cfc282e..5403442b 100644 --- a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts @@ -11,6 +11,8 @@ group = "com.iqser.red.service" java.sourceCompatibility = JavaVersion.VERSION_17 java.targetCompatibility = JavaVersion.VERSION_17 +val persistenceServiceVersion by rootProject.extra { "2.612.0-RED10072.1" } + pmd { isConsoleOutput = true } @@ -65,3 +67,13 @@ repositories { } } } + +allprojects { + tasks.withType { + options { + this as StandardJavadocDocletOptions + addBooleanOption("Xdoclint:none", true) + addStringOption("Xmaxwarns", "1") + } + } +} diff --git a/redaction-service-v1/document/build.gradle.kts b/redaction-service-v1/document/build.gradle.kts index 1ed3484f..50187c0e 100644 --- a/redaction-service-v1/document/build.gradle.kts +++ b/redaction-service-v1/document/build.gradle.kts @@ -4,13 +4,12 @@ plugins { } description = "redaction-service-document" -val persistenceServiceVersion = "2.612.0-RED10072.1" -val layoutParserVersion = "newNode" + group = "com.knecon.fforesight" dependencies { - implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + implementation("com.iqser.red.service:persistence-service-internal-api-v1:${rootProject.extra.get("persistenceServiceVersion")}") api("com.google.protobuf:protobuf-java-util:4.28.3") testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 863c188c..89e92e1f 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.651.0-RED9472.0" +val persistenceServiceVersion = "2.641.0" val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" @@ -31,21 +31,18 @@ configurations { } } -configurations.all { - resolutionStrategy { - force("com.google.protobuf:protobuf-java:4.27.1") - } -} - dependencies { - implementation(project(":document")) - implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } - implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") } - implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") - { - exclude(group = "com.knecon.fforesight", module = "tenant-commons") - } + implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } + implementation(project(":document")) + implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { + exclude(group = "org.springframework.boot") + exclude(group = "com.knecon.fforesight", module = "document") + } + implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { + exclude(group = "com.knecon.fforesight", module = "tenant-commons") + exclude(group = "com.knecon.fforesight", module = "document") + } implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}") implementation("com.iqser.red.commons:spring-commons:6.2.0") @@ -62,16 +59,14 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") - implementation("org.ahocorasick:ahocorasick:0.6.3") + implementation("org.ahocorasick:ahocorasick:0.9.0") implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") implementation("com.github.roklenarcic:aho-corasick:1.2") - implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") implementation("org.drools:drools-mvel:${droolsVersion}") implementation("org.kie:kie-spring:7.74.1.Final") - implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.locationtech.jts:jts-core:1.19.0") @@ -85,9 +80,11 @@ dependencies { implementation("org.apache.tomcat:tomcat-websocket:${tomcatVersion}") implementation("org.apache.tomcat.embed:tomcat-embed-core:${tomcatVersion}") + implementation("org.liquibase:liquibase-core:4.29.2") // Needed to be set explicit, otherwise spring dependency management sets it to 4.20.0 + implementation("org.liquibase.ext:liquibase-mongodb:4.29.2") + implementation("net.logstash.logback:logstash-logback-encoder:7.4") api("ch.qos.logback:logback-classic") - api("com.knecon.fforesight:swagger-commons:0.7.0") implementation("org.reflections:reflections:0.10.2") @@ -104,10 +101,8 @@ dependencies { testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}") testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}") testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") { - exclude( - group = "com.iqser.red.service", - module = "persistence-service-shared-api-v1" - ) + exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") + exclude(group = "com.knecon.fforesight", module = "document") } testImplementation("com.pdftron:PDFNet:10.11.0") } @@ -131,9 +126,9 @@ tasks.named("bootBuildImage") { environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ") environment.put( - "BPE_APPEND_JAVA_TOOL_OPTIONS", - "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5" + "BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5" ) + environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE! if (project.hasProperty("buildbootDockerHostNetwork")) { @@ -188,13 +183,19 @@ tasks.register("generateJavaDoc", Javadoc::class) { dependsOn("compileJava") dependsOn("delombok") classpath = project.sourceSets["main"].runtimeClasspath - source = fileTree("${buildDir}/generated/sources/delombok/java/main") { + val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { include(droolsImports) } - destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "") + val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { + include(droolsImports) + } + source = documentFiles + mainFiles + + setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "")) options.memberLevel = JavadocMemberLevel.PUBLIC (options as StandardJavadocDocletOptions).apply { title = "API Documentation for Redaction Service ${project.version}" } } + diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index cf0f1900..1ec1dd95 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -1477,11 +1477,9 @@ public class EntityCreationService { private void addListenerToEntity(TextEntity textEntity) { - if(kieSessionUpdater != null) { + if (kieSessionUpdater != null) { textEntity.addEntityEventListener(kieSessionUpdater); } } - - } -- 2.47.2 From 077e4d0ee1f171b9aedc7cbf2e8bb5682396cd8b Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 19 Dec 2024 14:13:57 +0100 Subject: [PATCH 103/108] RED-9742: fix document import --- .../redaction-service-api-v1/build.gradle.kts | 3 +-- .../redaction-service-server-v1/build.gradle.kts | 4 ++-- .../src/test/resources/files/syngenta | 2 +- redaction-service-v1/rules-management/build.gradle.kts | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 3c05f811..87ec3822 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,12 +4,11 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.651.0-RED9472.0" +val persistenceServiceVersion = "2.631.0" dependencies { implementation("org.springframework:spring-web:6.0.12") implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") - api("com.knecon.fforesight:swagger-commons:0.7.0") } publishing { diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 89e92e1f..f5f0d918 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.641.0" +val persistenceServiceVersion = rootProject.extra.get("persistenceServiceVersion") val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" @@ -191,7 +191,7 @@ tasks.register("generateJavaDoc", Javadoc::class) { } source = documentFiles + mainFiles - setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "")) + setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "javadoc")) options.memberLevel = JavadocMemberLevel.PUBLIC (options as StandardJavadocDocletOptions).apply { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta index 57e6e0dd..5705cc07 160000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta @@ -1 +1 @@ -Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7 +Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421 diff --git a/redaction-service-v1/rules-management/build.gradle.kts b/redaction-service-v1/rules-management/build.gradle.kts index b005ed7e..2aaea40c 100644 --- a/redaction-service-v1/rules-management/build.gradle.kts +++ b/redaction-service-v1/rules-management/build.gradle.kts @@ -27,10 +27,10 @@ sourceSets { } dependencies { - implementation(project(mapOf("path" to ":redaction-service-server-v1"))) + implementation(project(":redaction-service-server-v1")) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") - implementation(project(":redaction-service-server-v1")) + implementation("com.github.javaparser:javaparser-core:3.25.3") implementation("org.drools:drools-drl-parser:8.41.0.Final") implementation("org.apache.commons:commons-csv:1.10.0") -- 2.47.2 From 3776f800053b10854a2d9fa95859c3161cb414d8 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 19 Dec 2024 14:25:08 +0100 Subject: [PATCH 104/108] RED-9472: seperation of system rules added feature branch persistence service version --- .../kotlin/com.iqser.red.service.java-conventions.gradle.kts | 2 +- redaction-service-v1/redaction-service-api-v1/build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts index 5403442b..4c4ecaa1 100644 --- a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts @@ -11,7 +11,7 @@ group = "com.iqser.red.service" java.sourceCompatibility = JavaVersion.VERSION_17 java.targetCompatibility = JavaVersion.VERSION_17 -val persistenceServiceVersion by rootProject.extra { "2.612.0-RED10072.1" } +val persistenceServiceVersion by rootProject.extra { "2.651.0-RED9472.1" } pmd { isConsoleOutput = true diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 87ec3822..32fe813f 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.631.0" +val persistenceServiceVersion = rootProject.extra.get("persistenceServiceVersion") dependencies { implementation("org.springframework:spring-web:6.0.12") -- 2.47.2 From 9395b5ad6bfa222e9376e0e349e6d489b27c89d5 Mon Sep 17 00:00:00 2001 From: yhampe Date: Fri, 20 Dec 2024 09:23:49 +0100 Subject: [PATCH 105/108] RED-9472: seperation of system rules fixed checkstyle --- .../service/redaction/v1/server/model/RuleMergingResult.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java index b2cb1b27..c69423fb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RuleMergingResult.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.model; -import io.swagger.v3.oas.annotations.media.Schema; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -10,15 +9,11 @@ import lombok.NoArgsConstructor; @AllArgsConstructor @NoArgsConstructor @Builder -@Schema(description = "Object containing a string of Drools rules.") public class RuleMergingResult { - @Schema(description = "The merged rules.") private String mergedRules; - @Schema(description = "the length of added imports from sytemRules") private int addedImportsOffset; - @Schema(description = "the length of added globals from sytemRules") private int addedGlobalsOffset; } -- 2.47.2 From b48f448c24725cd3a4d03dbdc6d0c6f36cf3c242 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 20 Jan 2025 12:49:33 +0100 Subject: [PATCH 106/108] RED-9472: seperation of system rules working on failing tests --- .../redaction-service-server-v1/build.gradle.kts | 2 -- 1 file changed, 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index f5f0d918..f7311258 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -37,11 +37,9 @@ dependencies { implementation(project(":document")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") - exclude(group = "com.knecon.fforesight", module = "document") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") - exclude(group = "com.knecon.fforesight", module = "document") } implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}") -- 2.47.2 From 5a6746c5b2a9bcfa64289066c703f8e5e92b6979 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Mon, 20 Jan 2025 13:29:01 +0100 Subject: [PATCH 107/108] Potential fix for failing pipeline --- .../document/build.gradle.kts | 4 +- .../redaction-service-api-v1/build.gradle.kts | 4 +- .../build.gradle.kts | 62 ++++++++++--------- .../src/test/resources/files/syngenta | 2 +- 4 files changed, 39 insertions(+), 33 deletions(-) diff --git a/redaction-service-v1/document/build.gradle.kts b/redaction-service-v1/document/build.gradle.kts index 50187c0e..51dd3bb7 100644 --- a/redaction-service-v1/document/build.gradle.kts +++ b/redaction-service-v1/document/build.gradle.kts @@ -9,7 +9,9 @@ description = "redaction-service-document" group = "com.knecon.fforesight" dependencies { - implementation("com.iqser.red.service:persistence-service-internal-api-v1:${rootProject.extra.get("persistenceServiceVersion")}") + implementation("com.iqser.red.service:persistence-service-internal-api-v1:${rootProject.extra.get("persistenceServiceVersion")}") { + exclude(group = "com.knecon.fforesight", module = "document") + } api("com.google.protobuf:protobuf-java-util:4.28.3") testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 32fe813f..b8511726 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -8,7 +8,9 @@ val persistenceServiceVersion = rootProject.extra.get("persistenceServiceVersion dependencies { implementation("org.springframework:spring-web:6.0.12") - implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") + implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { + exclude(group = "com.knecon.fforesight", module = "document") + } } publishing { diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index f7311258..ab907ffd 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -37,9 +37,11 @@ dependencies { implementation(project(":document")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") + exclude(group = "com.knecon.fforesight", module = "document") } implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}") { exclude(group = "com.knecon.fforesight", module = "tenant-commons") + exclude(group = "com.knecon.fforesight", module = "document") } implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}") implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}") @@ -150,44 +152,44 @@ tasks.named("bootBuildImage") { } } -fun parseDroolsImports(vararg droolsFilePaths: String): List { - val imports = mutableListOf() - val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);") - val desiredPrefix = "com.iqser.red.service.redaction.v1" - - droolsFilePaths.forEach { filePath -> - File(filePath).forEachLine { line -> - importPattern.find(line)?.let { matchResult -> - val importPath = matchResult.groupValues[1].trim() - if (importPath.startsWith(desiredPrefix)) { - val formattedPath = importPath.replace('.', '/') - imports.add("$formattedPath.java") - } - } - } - } - - return imports -} +//fun parseDroolsImports(vararg droolsFilePaths: String): List { +// val imports = mutableListOf() +// val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);") +// val desiredPrefix = "com.iqser.red.service.redaction.v1" +// +// droolsFilePaths.forEach { filePath -> +// File(filePath).forEachLine { line -> +// importPattern.find(line)?.let { matchResult -> +// val importPath = matchResult.groupValues[1].trim() +// if (importPath.startsWith(desiredPrefix)) { +// val formattedPath = importPath.replace('.', '/') +// imports.add("$formattedPath.java") +// } +// } +// } +// } +// +// return imports +//} // Combine imports from both drools files -val droolsImports = parseDroolsImports( - "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl", - "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl" -) +//val droolsImports = parseDroolsImports( +// "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl", +// "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl" +//) tasks.register("generateJavaDoc", Javadoc::class) { dependsOn("compileJava") dependsOn("delombok") classpath = project.sourceSets["main"].runtimeClasspath - val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { - include(droolsImports) - } - val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { - include(droolsImports) - } - source = documentFiles + mainFiles +// val documentFiles = fileTree("${project(":document").layout.buildDirectory.get()}/generated/sources/delombok/java/main") { +// include(droolsImports) +// } +// val mainFiles = fileTree("${layout.buildDirectory.get()}/generated/sources/delombok/java/main") { +// include(droolsImports) +// } +// source = documentFiles + mainFiles setDestinationDir(file(project.findProperty("javadocDestinationDir")?.toString() ?: "javadoc")) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta index 5705cc07..57e6e0dd 160000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta @@ -1 +1 @@ -Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421 +Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7 -- 2.47.2 From f03b0a38f1098f49e00fa246b6db0b7f5fbda96c Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 22 Jan 2025 09:48:02 +0100 Subject: [PATCH 108/108] RED-9472: seperation of system rules added some logs for investigation --- .../redaction/v1/server/controller/RuleBuilderController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index a54de98a..ad2f7b83 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -99,6 +99,7 @@ public class RuleBuilderController implements RuleBuilderResource { } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } + log.info("response body: " + rulesUploadResponse); log.info("response: " + new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK)); return new ResponseEntity<>(rulesUploadResponse, HttpStatus.OK); } -- 2.47.2