Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-7052 Solved KieContainer race condition

Browse Source
This commit is contained in:
Timo Bejan 2023-08-02 23:17:51 +03:00
parent 8e63be5752
commit bd30e301e0
12 changed files with 82 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -69,3 +69,7 @@ build/
**/.DS_Store
**/classpath-data.json
**/dependencies-and-licenses-overview.txt
gradle.properties
gradlew
gradlew.bat
gradle/

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/KieWrapper.java Normal file
View File

@ -0,0 +1,7 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import org.kie.api.runtime.KieContainer;
public record KieWrapper( KieContainer container,long rulesVersion) {
}

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantRules.java
View File

@ -1,15 +0,0 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class TenantRules {
private Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
}

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -79,9 +79,8 @@ public class AnalyzeService {
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
log.info("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var wrapper = droolsExecutionService.getLatestKieContainer(analyzeRequest.getDossierTemplateId());
log.info("Updated Rules to Version {} for file {} in dossier {}", wrapper.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (analyzeRequest.getManualRedactions() != null) {
entityRedactionService.addManualAddRedactionEntities(analyzeRequest.getManualRedactions().getEntriesToAdd(), document);
@ -90,7 +89,7 @@ public class AnalyzeService {
entityRedactionService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities);
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, wrapper.container(), analyzeRequest, nerEntities);
log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId());
@ -102,7 +101,7 @@ public class AnalyzeService {
toSimplifiedSectionText(legalBasis),
dictionary.getVersion().getDossierTemplateVersion(),
dictionary.getVersion().getDossierVersion(),
rulesVersion,
wrapper.rulesVersion(),
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
@ -153,9 +152,9 @@ public class AnalyzeService {
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
var wrapper = droolsExecutionService.getLatestKieContainer(analyzeRequest.getDossierTemplateId());
log.info("Updated Rules to version {} for file {} in dossier {}",
droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()),
wrapper.rulesVersion(),
analyzeRequest.getFileId(),
analyzeRequest.getDossierId());
@ -170,7 +169,7 @@ public class AnalyzeService {
sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, wrapper.container(), analyzeRequest, nerEntities);
log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId());

125
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -7,40 +7,49 @@ import static java.util.stream.Collectors.toList;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collector;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.drools.compiler.kie.builder.impl.InternalKieServices;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.builder.ReleaseId;
import org.kie.api.runtime.KieContainer;
import org.kie.api.runtime.KieSession;
import org.kie.api.runtime.rule.QueryResults;
import org.kie.api.runtime.rule.QueryResultsRow;
import org.springframework.stereotype.Service;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import com.google.common.io.ByteStreams;
import com.google.common.primitives.Bytes;
import com.google.common.primitives.Longs;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel;
@ -55,28 +64,9 @@ import lombok.extern.slf4j.Slf4j;
public class DroolsExecutionService {
final RulesClient rulesClient;
Map<String, KieContainer> kieContainers = new HashMap<>();
final Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
final EntityEnrichmentService entityEnrichmentService;
public KieContainer getKieContainer(String dossierTemplateId) {
KieContainer container = kieContainers.get(dossierTemplateId);
if (container == null) {
return createOrUpdateKieContainer(dossierTemplateId);
} else {
return container;
}
}
public void invalidateKieContainerCache() {
// TODO: fix this cache!
// This cache fails 5 tests from redactionIntegrationTest
kieContainers = new HashMap<>();
}
@Timed("redactmanager_executeRules")
public List<FileAttribute> executeRules(KieContainer kieContainer,
Document document,
@ -159,79 +149,76 @@ public class DroolsExecutionService {
}
public KieContainer updateRules(String dossierTemplateId) {
public KieWrapper getLatestKieContainer(String dossierTemplateId) {
long version = rulesClient.getVersion(dossierTemplateId);
Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
if (rulesVersion == null) {
rulesVersion = -1L;
}
if (version > rulesVersion) {
rulesVersionPerDossierTemplateId.put(dossierTemplateId, version);
return createOrUpdateKieContainer(dossierTemplateId);
}
return getKieContainer(dossierTemplateId);
return new KieWrapper(getKieContainer(dossierTemplateId, version), version);
}
private KieContainer createOrUpdateKieContainer(String dossierTemplateId) {
private ReleaseId getReleaseId(String dossierTemplate, long version) {
KieServices kieServices = KieServices.Factory.get();
return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate, String.format("1.%d", version));
}
private KieContainer getKieContainer(String dossierTemplateId, long version) {
KieServices kieServices = KieServices.Factory.get();
try {
var rules = rulesClient.getRules(dossierTemplateId);
if (rules == null || StringUtils.isEmpty(rules.getValue())) {
throw new RuntimeException("Rules cannot be empty.");
}
KieServices kieServices = KieServices.Factory.get();
KieModule kieModule = getKieModule(dossierTemplateId, rules.getValue(), kieServices);
var container = kieContainers.get(dossierTemplateId);
if (container != null) {
container.updateToVersion(kieModule.getReleaseId());
return container;
}
container = kieServices.newKieContainer(kieModule.getReleaseId());
kieContainers.put(dossierTemplateId, container);
return container;
return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version));
} catch (Exception e) {
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
registerNewKieContainerVersion(dossierTemplateId, version);
return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version));
}
}
private KieModule getKieModule(String dossierTemplateId, String rules, KieServices kieServices) {
private void registerNewKieContainerVersion(String dossierTemplateId, long version) {
var rules = rulesClient.getRules(dossierTemplateId);
if (rules == null || StringUtils.isEmpty(rules.getValue())) {
throw new RuntimeException("Rules cannot be empty.");
}
registerNewKieContainerVersion(dossierTemplateId, version, rules.getValue());
}
private void registerNewKieContainerVersion(String dossierTemplateId, long version, String rules) {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.generateAndWritePomXML(getReleaseId(dossierTemplateId, version));
InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
return kieBuilder.getKieModule();
}
public void testRules(String rules) {
KieServices kieServices = KieServices.Factory.get();
KieModule kieModule = getKieModule("test-rules", rules, kieServices);
var container = kieServices.newKieContainer(kieModule.getReleaseId());
container.newKieSession();
container.dispose();
}
var versionId = System.currentTimeMillis();
var testRules = "test-rules";
registerNewKieContainerVersion(testRules, versionId, rules);
var releaseId = getReleaseId(testRules, versionId);
try {
KieServices kieServices = KieServices.Factory.get();
var containerId = UUID.randomUUID().toString();
var container = kieServices.newKieContainer(containerId, releaseId);
container.newKieSession();
container.dispose();
public long getRulesVersion(String dossierTemplateId) {
Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
if (rulesVersion == null) {
return -1;
} catch (Exception e) {
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
}
return rulesVersion;
}
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -143,7 +143,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
loadTypeForTest();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -70,7 +70,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
loadDictionaryForTest();

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -116,7 +116,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@BeforeEach
public void invalidateCaches() {
droolsExecutionService.invalidateKieContainerCache();
// droolsExecutionService.invalidateKieContainerCache();
}
@ -125,7 +125,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
loadDictionaryForTest();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -60,7 +60,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
loadDictionaryForTest();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -263,7 +263,7 @@ public class RulesTest {
objectMapper.registerModule(new JavaTimeModule());
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
loadDictionaryForTest();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -98,7 +98,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
entityCreationService = new EntityCreationService(entityEnrichmentService);
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
loadDictionaryForTest();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java
View File

@ -46,7 +46,7 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest {
public void stubClients() {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
loadDictionaryForTest();
loadTypeForTest();