From 282ee7652d2a09a6fa8b640bc8fd467dad1183d2 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 17 Apr 2023 13:25:45 +0200 Subject: [PATCH] RED-6369: Rules Refactor * marked entities as resized if a resizeRedaction with same id exists --- .../document/graph/entity/EntityNode.java | 6 ++ .../service/DroolsExecutionService.java | 72 ++++--------------- .../service/analyze/AnalyzeService.java | 13 +++- .../EntityRedactionService.java | 31 ++++---- .../src/test/resources/drools/rules.drl | 14 ++++ 5 files changed, 64 insertions(+), 72 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java index 5195c658..93911b70 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java @@ -139,6 +139,12 @@ public class EntityNode { } + public boolean matchesResizedAnnotationId(String resizedAnnotationId) { + + return getEntityPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(resizedAnnotationId)); + } + + @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index c460d04d..6921ae84 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -8,12 +8,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.kie.api.KieServices; @@ -27,6 +21,7 @@ import org.kie.api.runtime.rule.QueryResultsRow; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; @@ -39,20 +34,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dict import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; -import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; -import com.iqser.red.service.redaction.v1.server.redaction.model.Section; -import com.iqser.red.service.redaction.v1.server.redaction.model.TenantRules; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; - -import io.micrometer.core.annotation.Timed; -import jakarta.annotation.PostConstruct; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; @Service @RequiredArgsConstructor @@ -62,24 +43,7 @@ public class DroolsExecutionService { private final Map kieContainers = new HashMap<>(); - private final RedactionServiceSettings settings; - - private LoadingCache tenantRulesCache; - - - @PostConstruct - protected void createCache() { - - tenantRulesCache = CacheBuilder.newBuilder() - .maximumSize(settings.getDictionaryCacheMaximumSize()) - .expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS) - .build(new CacheLoader<>() { - public TenantRules load(String key) { - - return new TenantRules(); - } - }); - } + private final Map rulesVersionPerDossierTemplateId = new HashMap<>(); private final EntityCreationService entityCreationService; @@ -96,7 +60,11 @@ public class DroolsExecutionService { @Timed("redactmanager_executeRules") - public List executeRules(KieContainer kieContainer, DocumentGraph document, Dictionary dictionary, List fileAttributes) { + public List executeRules(KieContainer kieContainer, + DocumentGraph document, + Dictionary dictionary, + List fileAttributes, + Set resizeRedactions) { KieSession kieSession = kieContainer.newKieSession(); kieSession.setGlobal("document", document); @@ -107,6 +75,7 @@ public class DroolsExecutionService { document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode())); document.getPages().forEach(kieSession::insert); fileAttributes.forEach(kieSession::insert); + resizeRedactions.forEach(kieSession::insert); kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); kieSession.fireAllRules(); @@ -120,7 +89,8 @@ public class DroolsExecutionService { DocumentGraph document, List sectionsToReanalyze, Dictionary dictionary, - List fileAttributes) { + List fileAttributes, + Set resizeRedactions) { KieSession kieSession = kieContainer.newKieSession(); kieSession.setGlobal("document", document); @@ -128,9 +98,11 @@ public class DroolsExecutionService { kieSession.setGlobal("dictionary", dictionary); document.getEntities().forEach(kieSession::insert); + sectionsToReanalyze.forEach(kieSession::insert); sectionsToReanalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert); document.getPages().forEach(kieSession::insert); fileAttributes.forEach(kieSession::insert); + resizeRedactions.forEach(kieSession::insert); kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); kieSession.fireAllRules(); @@ -178,13 +150,13 @@ public class DroolsExecutionService { public KieContainer updateRules(String dossierTemplateId) { long version = rulesClient.getVersion(dossierTemplateId); - Long rulesVersion = getVersionForDossierTemplate(dossierTemplateId); + Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId); if (rulesVersion == null) { rulesVersion = -1L; } if (version > rulesVersion) { - setRulesVersionForDossierTemplate(dossierTemplateId, version); + rulesVersionPerDossierTemplateId.put(dossierTemplateId, version); return createOrUpdateKieContainer(dossierTemplateId); } return getKieContainer(dossierTemplateId); @@ -243,25 +215,11 @@ public class DroolsExecutionService { public long getRulesVersion(String dossierTemplateId) { - Long rulesVersion = getVersionForDossierTemplate(dossierTemplateId); + Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId); if (rulesVersion == null) { return -1; } return rulesVersion; } - - @SneakyThrows - private Long getVersionForDossierTemplate(String dossierTemplateId) { - - return tenantRulesCache.get(TenantContext.getTenantId()).getRulesVersionPerDossierTemplateId().get(dossierTemplateId); - } - - - @SneakyThrows - private void setRulesVersionForDossierTemplate(String dossierTemplateId, long version) { - - tenantRulesCache.get(TenantContext.getTenantId()).getRulesVersionPerDossierTemplateId().put(dossierTemplateId, version); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index 0e226e96..6cfbc963 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -185,7 +185,12 @@ public class AnalyzeService { KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); + Set addedFileAttributes = entityRedactionService.addRuleEntitiesToGraphAndReturnAddedFileAttributes(dictionary, + documentGraph, + sectionsToReAnalyse, + kieContainer, + analyzeRequest, + nerEntities); List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId()); @@ -220,7 +225,11 @@ public class AnalyzeService { Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); entityRedactionService.addDictionaryEntities(dictionary, documentGraph); - Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities); + Set addedFileAttributes = entityRedactionService.addRuleEntitiesToGraphAndReturnAddedFileAttributes(dictionary, + documentGraph, + kieContainer, + analyzeRequest, + nerEntities); List redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index 482c3bdd..cb01fcc6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -33,29 +33,34 @@ public class EntityRedactionService { EntityCreationService entityCreationService; - public Set addRuleEntities(Dictionary dictionary, - DocumentGraph documentGraph, - KieContainer kieContainer, - AnalyzeRequest analyzeRequest, - NerEntities nerEntities) { + public Set addRuleEntitiesToGraphAndReturnAddedFileAttributes(Dictionary dictionary, + DocumentGraph documentGraph, + KieContainer kieContainer, + AnalyzeRequest analyzeRequest, + NerEntities nerEntities) { - List allFileAttributes = droolsExecutionService.executeRules(kieContainer, documentGraph, dictionary, analyzeRequest.getFileAttributes()); + List allFileAttributes = droolsExecutionService.executeRules(kieContainer, + documentGraph, + dictionary, + analyzeRequest.getFileAttributes(), + analyzeRequest.getManualRedactions().getResizeRedactions()); return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet()); } - public Set addRuleEntities(Dictionary dictionary, - DocumentGraph documentGraph, - List sectionsToReanalyze, - KieContainer kieContainer, - AnalyzeRequest analyzeRequest, - NerEntities nerEntities) { + public Set addRuleEntitiesToGraphAndReturnAddedFileAttributes(Dictionary dictionary, + DocumentGraph documentGraph, + List sectionsToReanalyze, + KieContainer kieContainer, + AnalyzeRequest analyzeRequest, + NerEntities nerEntities) { List allFileAttributes = droolsExecutionService.executeRules(kieContainer, documentGraph, sectionsToReanalyze, dictionary, - analyzeRequest.getFileAttributes()); + analyzeRequest.getFileAttributes(), + analyzeRequest.getManualRedactions().getResizeRedactions()); return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index bc770eb4..0ac78b23 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; global DocumentGraph document global EntityCreationService entityCreationService @@ -31,6 +32,19 @@ query "getFileAttributes" $fileAttribute: FileAttribute() end +// --------------------------------------- apply resize redactions ------------------------------------------------------------------- + +rule "apply resize redactions" + salience 200 + no-loop true + + when + $resizeRedaction: ManualResizeRedaction($id: annotationId) + $resizedEntity: EntityNode(matchesResizedAnnotationId($id)) + then + $resizedEntity.setResized(true); + update($resizedEntity); + end // --------------------------------------- merging rules ------------------------------------------------------------------- rule "merge intersecting Entities of same type"