diff --git a/.gitignore b/.gitignore index 2594dd52..9cf1cfa7 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,7 @@ build/ **/.DS_Store **/classpath-data.json **/dependencies-and-licenses-overview.txt +gradle.properties +gradlew +gradlew.bat +gradle/ diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/KieWrapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/KieWrapper.java new file mode 100644 index 00000000..e9765698 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/KieWrapper.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import org.kie.api.runtime.KieContainer; + +public record KieWrapper( KieContainer container,long rulesVersion) { + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantRules.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantRules.java deleted file mode 100644 index eff0a617..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantRules.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; - -import java.util.HashMap; -import java.util.Map; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class TenantRules { - - private Map rulesVersionPerDossierTemplateId = new HashMap<>(); - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 69fba0a3..b34b141d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -79,9 +79,8 @@ public class AnalyzeService { Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); - log.info("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + var wrapper = droolsExecutionService.getLatestKieContainer(analyzeRequest.getDossierTemplateId()); + log.info("Updated Rules to Version {} for file {} in dossier {}", wrapper.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (analyzeRequest.getManualRedactions() != null) { entityRedactionService.addManualAddRedactionEntities(analyzeRequest.getManualRedactions().getEntriesToAdd(), document); @@ -90,7 +89,7 @@ public class AnalyzeService { entityRedactionService.addDictionaryEntities(dictionary, document); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities); + Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, wrapper.container(), analyzeRequest, nerEntities); log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); @@ -102,7 +101,7 @@ public class AnalyzeService { toSimplifiedSectionText(legalBasis), dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(), - rulesVersion, + wrapper.rulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); List importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), @@ -153,9 +152,9 @@ public class AnalyzeService { NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); + var wrapper = droolsExecutionService.getLatestKieContainer(analyzeRequest.getDossierTemplateId()); log.info("Updated Rules to version {} for file {} in dossier {}", - droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()), + wrapper.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -170,7 +169,7 @@ public class AnalyzeService { sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); + Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, wrapper.container(), analyzeRequest, nerEntities); log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 6e1ccdf2..1b0bf58e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -7,40 +7,49 @@ import static java.util.stream.Collectors.toList; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collection; import java.util.Comparator; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; +import java.util.UUID; import java.util.stream.Collector; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.drools.compiler.kie.builder.impl.InternalKieServices; import org.kie.api.KieServices; import org.kie.api.builder.KieBuilder; import org.kie.api.builder.KieFileSystem; -import org.kie.api.builder.KieModule; +import org.kie.api.builder.ReleaseId; import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieSession; import org.kie.api.runtime.rule.QueryResults; import org.kie.api.runtime.rule.QueryResultsRow; import org.springframework.stereotype.Service; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteStreams; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Longs; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService; +import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; +import com.iqser.red.service.redaction.v1.server.redaction.model.KieWrapper; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.knecon.fforesight.tenantcommons.TenantContext; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -55,28 +64,9 @@ import lombok.extern.slf4j.Slf4j; public class DroolsExecutionService { final RulesClient rulesClient; - Map kieContainers = new HashMap<>(); - final Map rulesVersionPerDossierTemplateId = new HashMap<>(); + final EntityEnrichmentService entityEnrichmentService; - - public KieContainer getKieContainer(String dossierTemplateId) { - - KieContainer container = kieContainers.get(dossierTemplateId); - if (container == null) { - return createOrUpdateKieContainer(dossierTemplateId); - } else { - return container; - } - } - - public void invalidateKieContainerCache() { - // TODO: fix this cache! - // This cache fails 5 tests from redactionIntegrationTest - kieContainers = new HashMap<>(); - } - - @Timed("redactmanager_executeRules") public List executeRules(KieContainer kieContainer, Document document, @@ -159,79 +149,76 @@ public class DroolsExecutionService { } - public KieContainer updateRules(String dossierTemplateId) { + public KieWrapper getLatestKieContainer(String dossierTemplateId) { long version = rulesClient.getVersion(dossierTemplateId); - Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId); - if (rulesVersion == null) { - rulesVersion = -1L; - } - if (version > rulesVersion) { - rulesVersionPerDossierTemplateId.put(dossierTemplateId, version); - return createOrUpdateKieContainer(dossierTemplateId); - } - return getKieContainer(dossierTemplateId); + return new KieWrapper(getKieContainer(dossierTemplateId, version), version); } - private KieContainer createOrUpdateKieContainer(String dossierTemplateId) { + private ReleaseId getReleaseId(String dossierTemplate, long version) { + KieServices kieServices = KieServices.Factory.get(); + return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate, String.format("1.%d", version)); + } + + + private KieContainer getKieContainer(String dossierTemplateId, long version) { + + KieServices kieServices = KieServices.Factory.get(); try { - - var rules = rulesClient.getRules(dossierTemplateId); - if (rules == null || StringUtils.isEmpty(rules.getValue())) { - throw new RuntimeException("Rules cannot be empty."); - } - - KieServices kieServices = KieServices.Factory.get(); - KieModule kieModule = getKieModule(dossierTemplateId, rules.getValue(), kieServices); - - var container = kieContainers.get(dossierTemplateId); - if (container != null) { - container.updateToVersion(kieModule.getReleaseId()); - return container; - } - - container = kieServices.newKieContainer(kieModule.getReleaseId()); - kieContainers.put(dossierTemplateId, container); - return container; + return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version)); } catch (Exception e) { - throw new RulesValidationException("Could not update rules: " + e.getMessage(), e); + + registerNewKieContainerVersion(dossierTemplateId, version); + return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version)); } } - private KieModule getKieModule(String dossierTemplateId, String rules, KieServices kieServices) { + private void registerNewKieContainerVersion(String dossierTemplateId, long version) { + var rules = rulesClient.getRules(dossierTemplateId); + if (rules == null || StringUtils.isEmpty(rules.getValue())) { + throw new RuntimeException("Rules cannot be empty."); + } + + registerNewKieContainerVersion(dossierTemplateId, version, rules.getValue()); + + } + + + private void registerNewKieContainerVersion(String dossierTemplateId, long version, String rules) { + + KieServices kieServices = KieServices.Factory.get(); KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); + kieFileSystem.generateAndWritePomXML(getReleaseId(dossierTemplateId, version)); InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8)); kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources().newInputStreamResource(input)); KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); kieBuilder.buildAll(); - return kieBuilder.getKieModule(); } public void testRules(String rules) { - KieServices kieServices = KieServices.Factory.get(); - KieModule kieModule = getKieModule("test-rules", rules, kieServices); - var container = kieServices.newKieContainer(kieModule.getReleaseId()); - container.newKieSession(); - container.dispose(); - } + var versionId = System.currentTimeMillis(); + var testRules = "test-rules"; + registerNewKieContainerVersion(testRules, versionId, rules); + var releaseId = getReleaseId(testRules, versionId); + try { + KieServices kieServices = KieServices.Factory.get(); + var containerId = UUID.randomUUID().toString(); + var container = kieServices.newKieContainer(containerId, releaseId); + container.newKieSession(); + container.dispose(); - - public long getRulesVersion(String dossierTemplateId) { - - Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId); - if (rulesVersion == null) { - return -1; + } catch (Exception e) { + throw new RulesValidationException("Could not update rules: " + e.getMessage(), e); } - return rulesVersion; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 9a882f88..9b356a89 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -143,7 +143,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); loadTypeForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index fb277871..be263ca9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -70,7 +70,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); loadDictionaryForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 3bb3842a..d53d25dd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -116,7 +116,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @BeforeEach public void invalidateCaches() { - droolsExecutionService.invalidateKieContainerCache(); + // droolsExecutionService.invalidateKieContainerCache(); } @@ -125,7 +125,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); loadDictionaryForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 68f8060a..77602279 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -60,7 +60,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); loadDictionaryForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 3b22caa0..8509c614 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -263,7 +263,7 @@ public class RulesTest { objectMapper.registerModule(new JavaTimeModule()); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); loadDictionaryForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 9b44657b..b2a6fa01 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -98,7 +98,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration entityCreationService = new EntityCreationService(entityEnrichmentService); TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); loadDictionaryForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java index dac48779..f43bbacd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java @@ -46,7 +46,7 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest { public void stubClients() { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); loadDictionaryForTest(); loadTypeForTest();