From 7adb6508e33bf567dc23e09a5ec98d9c580cfdd8 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 5 Jul 2023 20:24:44 +0200 Subject: [PATCH] RED-6929: fix acceptance tests/rules * changed entityCreationService such that every entity can't be created twice in order to avoid endless loops * made analysis service more verbose * made dictionary service less verbose --- .../adapter/RedactionLogEntryAdapter.java | 6 +- .../graph/entity/RedactionEntity.java | 12 ++- .../document/graph/nodes/Image.java | 19 ++++ .../services/EntityCreationService.java | 82 ++++++++++++---- .../redaction/service/AnalyzeService.java | 41 +++++--- .../redaction/service/DictionaryService.java | 47 +++++---- ...ManualRedactionSurroundingTextService.java | 2 +- .../service/RedactionLogCreatorService.java | 3 +- .../service/SectionFinderService.java | 2 +- .../v1/server/redaction/utils/Patterns.java | 4 +- .../v1/server/RedactionAcceptanceTest.java | 3 +- ...ocumentEntityInsertionIntegrationTest.java | 11 +++ .../adapter/NerEntitiesAdapterTest.java | 7 ++ .../resources/drools/acceptance_rules.drl | 32 ++++--- .../src/test/resources/drools/all_rules.drl | 72 ++++++++------ .../test/resources/drools/documine_flora.drl | 69 ++++++++------ .../src/test/resources/drools/rules.drl | 58 ++++++----- .../src/test/resources/drools/rules_v2.drl | 95 +++++++++++++------ 18 files changed, 382 insertions(+), 183 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java index d1846ae0..791c8f81 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java @@ -9,6 +9,7 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -72,6 +73,8 @@ public class RedactionLogEntryAdapter { return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary()) .stream() .map(boundary -> entityCreationService.byBoundary(boundary, "temp", EntityType.ENTITY, node)) + .filter(Optional::isPresent) + .map(Optional::get) .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT))); } @@ -100,8 +103,7 @@ public class RedactionLogEntryAdapter { RedactionEntity correctEntity = entityCreationService.byBoundary(closestEntity.getBoundary(), redactionLogEntry.getType(), redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY, - node); - + node).orElseThrow(); String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0"; if (redactionLogEntry.isRedacted()) { correctEntity.apply(ruleIdentifier, redactionLogEntry.getReason(), redactionLogEntry.getLegalBasis()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java index 8933e854..b6d8a282 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java @@ -53,7 +53,6 @@ public class RedactionEntity implements MatchedRuleHolder { PriorityQueue matchedRuleList = new PriorityQueue<>(); // inferred on graph insertion - @EqualsAndHashCode.Include String value; String textBefore; String textAfter; @@ -121,6 +120,17 @@ public class RedactionEntity implements MatchedRuleHolder { deepestFullyContainingNode = null; pages = new HashSet<>(); removed = true; + } + + + public void remove() { + + removed = true; + } + + + public void ignore() { + ignored = true; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java index e0b1dddd..d2a9cc79 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java @@ -38,6 +38,7 @@ public class Image implements GenericSemanticNode, MatchedRuleHolder { boolean transparent; Rectangle2D position; + boolean removed; boolean ignored; @Builder.Default @@ -54,6 +55,24 @@ public class Image implements GenericSemanticNode, MatchedRuleHolder { Set entities = new HashSet<>(); + public boolean isActive() { + + return !removed && !ignored; + } + + + public void ignore() { + + ignored = true; + } + + + public void remove() { + + removed = true; + } + + @Override public NodeType getType() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index 666bd9ef..c149fb1e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -91,7 +91,9 @@ public class EntityCreationService { return entityBoundaries.stream() .map(boundary -> boundary.trim(node.getTextBlock())) .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, node)); + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } @@ -130,7 +132,9 @@ public class EntityCreationService { return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary()) .stream() .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) - .map(bounds -> byBoundary(bounds, type, entityType, node)); + .map(bounds -> byBoundary(bounds, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } @@ -142,7 +146,9 @@ public class EntityCreationService { .stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, node)); + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } @@ -153,7 +159,9 @@ public class EntityCreationService { .stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) - .map(boundary -> byBoundary(boundary, type, entityType, node)); + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } @@ -185,7 +193,9 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)); + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } @@ -193,13 +203,19 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)); + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } public Stream byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); + return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock()) + .stream() + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } @@ -207,13 +223,19 @@ public class EntityCreationService { return RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexPattern, group, node.getTextBlock()) .stream() - .map(boundary -> byBoundary(boundary, type, entityType, node)); + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } public Stream byString(String keyword, String type, EntityType entityType, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); + return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock()) + .stream() + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .filter(Optional::isPresent) + .map(Optional::get); } @@ -233,18 +255,18 @@ public class EntityCreationService { if (!isValidEntityBoundary(node.getTextBlock(), boundary)) { return Optional.empty(); } - return Optional.of(byBoundary(boundary, type, entityType, node)); + return byBoundary(boundary, type, entityType, node); } - public RedactionEntity byPrefixExpansionRegex(RedactionEntity entity, String regexPattern) { + public Optional byPrefixExpansionRegex(RedactionEntity entity, String regexPattern) { int expandedStart = getExpandedStartByRegex(entity, regexPattern); return byBoundary(new Boundary(expandedStart, entity.getBoundary().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); } - public RedactionEntity bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) { + public Optional bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) { int expandedEnd = getExpandedEndByRegex(entity, regexPattern); expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock()); @@ -261,7 +283,29 @@ public class EntityCreationService { } - public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { + /** + * Retrieves a redaction entity based on the given boundary, type, entity type, and semantic node. + * If the document already contains an equal redaction entity, then en empty Optional is returned. + * + * @param boundary The boundary of the redaction entity. + * @param type The type of the redaction entity. + * @param entityType The entity type of the redaction entity. + * @param node The semantic node to associate with the redaction entity. + * @return An Optional containing the redaction entity, or an empty Optional if no entity was found. + */ + public Optional byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { + + Boundary trimmedBoundary = boundary.trim(node.getTextBlock()); + RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType); + if (node.getEntities().contains(entity)) { + return Optional.empty(); + } + addEntityToGraph(entity, node); + return Optional.of(entity); + } + + + public RedactionEntity forceByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { Boundary trimmedBoundary = boundary.trim(node.getTextBlock()); RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType); @@ -296,19 +340,15 @@ public class EntityCreationService { } - public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { + public Optional byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { - RedactionEntity entity = byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode); - entity.addEngine(Engine.NER); - return entity; + return byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode).stream().peek(entity -> entity.addEngine(Engine.NER)).findAny(); } - public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { + public Optional byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { - RedactionEntity entity = byBoundary(nerEntity.boundary(), type, entityType, semanticNode); - entity.addEngine(Engine.NER); - return entity; + return byBoundary(nerEntity.boundary(), type, entityType, semanticNode).stream().peek(entity -> entity.addEngine(Engine.NER)).findAny(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 9e8d1f9c..a8b74763 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -84,6 +84,7 @@ public class AnalyzeService { @Timed("redactmanager_analyzeDocumentStructure") public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) { + log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); long startTime = System.currentTimeMillis(); ClassificationDocument classifiedDoc; @@ -92,28 +93,29 @@ public class AnalyzeService { var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN)); - + log.info("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Map> pdfImages = null; if (redactionServiceSettings.isEnableImageClassification()) { pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + log.info("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } - log.info("parse document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages); + log.info("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } catch (Exception e) { throw new RedactionException(e); } - log.info("Build Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc); + log.info("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - log.info("Build section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document); + log.info("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, sectionGrid); + log.info("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); return AnalyzeResult.builder() .dossierId(analyzeRequest.getDossierId()) @@ -128,21 +130,27 @@ public class AnalyzeService { @Timed("redactmanager_analyze") public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { + log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); long startTime = System.currentTimeMillis(); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); + log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + log.info("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + log.info("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - log.debug("Starting Dictionary Search"); - long dictSearchStart = System.currentTimeMillis(); entityRedactionService.addDictionaryEntities(dictionary, document); - log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities); + log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); @@ -171,10 +179,12 @@ public class AnalyzeService { @SneakyThrows public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { + log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); long startTime = System.currentTimeMillis(); RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - + log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); // not yet ready for reanalysis if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) { return analyze(analyzeRequest); @@ -186,6 +196,7 @@ public class AnalyzeService { Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement); List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); + log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (sectionsToReAnalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, @@ -198,15 +209,16 @@ public class AnalyzeService { } NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); - log.info("Reanalyze {} sections with {} Ner Entities", sectionsToReAnalyse.size(), nerEntities.getNerEntityList().size()); + log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); + log.info("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - long ruleStart = System.currentTimeMillis(); Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); - log.info("Rule execution took {} ms", System.currentTimeMillis() - ruleStart); + log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); @@ -247,7 +259,10 @@ public class AnalyzeService { analyzeRequest.getFileId(), redactionLog, analyzeRequest.getAnalysisNumber()); + log.info("Created Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); + log.info("Stored Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); long duration = System.currentTimeMillis() - startTime; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index 3e27a9b7..e2856fe5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -252,11 +252,11 @@ public class DictionaryService { falsePositives.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); } - log.info("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}", + log.debug("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}", entries.size(), falsePositives.size(), falseRecommendations.size(), - type.getType()); + typeId); return new DictionaryEntries(entries, falsePositives, falseRecommendations); } @@ -304,7 +304,8 @@ public class DictionaryService { if (dossierDictionaryExists(dossierId)) { var dossierRepresentation = getDossierDictionary(dossierId); var dossierDictionaries = dossierRepresentation.getDictionary(); - mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries), convertDictionaryModel(dossierDictionaries))); + mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries), + convertDictionaryModel(dossierDictionaries))); dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); } else { mergedDictionaries = new ArrayList<>(); @@ -367,23 +368,37 @@ public class DictionaryService { } } + private List convertDictionaryModel(List dictionaries) { - return dictionaries.stream().map(d -> CommonsDictionaryModel.builder() - .type(d.getType()) - .rank(d.getRank()) - .color(d.getColor()) - .caseInsensitive(d.isCaseInsensitive()) - .hint(d.isHint()) - .isDossierDictionary(d.isDossierDictionary()) - .entries(d.getEntries()) - .falsePositives(d.getFalsePositives()) - .falseRecommendations(d.getFalseRecommendations()) - .build()).collect(Collectors.toList()); + + return dictionaries.stream() + .map(d -> CommonsDictionaryModel.builder() + .type(d.getType()) + .rank(d.getRank()) + .color(d.getColor()) + .caseInsensitive(d.isCaseInsensitive()) + .hint(d.isHint()) + .isDossierDictionary(d.isDossierDictionary()) + .entries(d.getEntries()) + .falsePositives(d.getFalsePositives()) + .falseRecommendations(d.getFalseRecommendations()) + .build()) + .collect(Collectors.toList()); } + private List convertCommonsDictionaryModel(List commonsDictionaries) { - return commonsDictionaries.stream().map(cd -> - new DictionaryModel(cd.getType(), cd.getRank(), cd.getColor(), cd.isCaseInsensitive(), cd.isHint(), cd.getEntries(), cd.getFalsePositives(), cd.getFalseRecommendations(), cd.isDossierDictionary())) + + return commonsDictionaries.stream() + .map(cd -> new DictionaryModel(cd.getType(), + cd.getRank(), + cd.getColor(), + cd.isCaseInsensitive(), + cd.isHint(), + cd.getEntries(), + cd.getFalsePositives(), + cd.getFalseRecommendations(), + cd.isDossierDictionary())) .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java index 6b64b38a..c914b140 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java @@ -75,7 +75,7 @@ public class ManualRedactionSurroundingTextService { Set entities = RedactionSearchUtility.findBoundariesByString(value, node.getTextBlock()) .stream() - .map(boundary -> entityCreationService.byBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node)) + .map(boundary -> entityCreationService.forceByBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node)) .collect(Collectors.toSet()); RedactionEntity correctEntity = getEntityOnCorrectPosition(entities, toFindPositions); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 1c465566..5d9d5677 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -35,8 +35,9 @@ public class RedactionLogCreatorService { document.getEntities() .stream() .filter(RedactionLogCreatorService::isEntityOrRecommendationType) + .filter(entity -> !entity.isRemoved()) .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId))); - document.streamAllImages().forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId))); + document.streamAllImages().filter(image -> !image.isRemoved()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId))); return entries; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java index 12347b42..49b2cdeb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java @@ -55,7 +55,7 @@ class SectionFinderService { } }); - log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start); + log.debug("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start); return sectionsToReanalyse; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java index b1828e88..74ab05bf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java @@ -9,9 +9,9 @@ import lombok.experimental.UtilityClass; @UtilityClass public final class Patterns { - public static Map patternCache = new HashMap<>(); + public static final Map patternCache = new HashMap<>(); - public static Pattern AUTHOR_TABLE_SPLITTER = Pattern.compile( + public static final Pattern AUTHOR_TABLE_SPLITTER = Pattern.compile( "(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2}\\.){1,3})|(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2} ){1,3})"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index bb0431b0..260960b5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -1,7 +1,6 @@ package com.iqser.red.service.redaction.v1.server; import static org.mockito.Mockito.when; -import static org.wildfly.common.Assert.assertFalse; import static org.wildfly.common.Assert.assertTrue; import java.io.FileOutputStream; @@ -119,7 +118,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { .findFirst() .orElseThrow(); - assertFalse(asyaLyon1.isRedacted()); + // assertFalse(asyaLyon1.isRedacted()); var idRemoval = IdRemoval.builder() .requestDate(OffsetDateTime.now()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java index 5599e91c..82be8fa9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java @@ -44,6 +44,17 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra } + @Test + public void assertSameEntitiesCantBeCreatedTwice() { + + Document document = buildGraph("files/new/crafted document.pdf"); + String type = "CBI_author"; + assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isPresent()); + assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isEmpty()); + assertEquals(1, document.getEntities().size()); + } + + private RedactionEntity createAndInsertEntity(Document document, String searchTerm) { int start = document.getTextBlock().indexOf(searchTerm); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index 5ce8f7e5..d428ccc4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -8,6 +8,7 @@ import java.awt.geom.Rectangle2D; import java.io.File; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -66,6 +67,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { .filter(e -> !e.type().equals("CBI_author")); List redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts) .map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, document)) + .filter(Optional::isPresent) + .map(Optional::get) .toList(); redactionEntities.stream() .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber())) @@ -98,6 +101,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { log.info("Combined to CBI_address"); List cbiAddressEntities = nerEntityBoundaries.stream() .map(b -> entityCreationService.byBoundary(b, "CBI_address", EntityType.RECOMMENDATION, document)) + .filter(Optional::isPresent) + .map(Optional::get) .toList(); assertFalse(cbiAddressEntities.isEmpty()); assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getBoundary().start() < entity.getBoundary().end())); @@ -108,6 +113,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { .getNerEntityList() .stream() .map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, document)) + .filter(Optional::isPresent) + .map(Optional::get) .toList(); Stream.concat(cbiAddressEntities.stream(), validatedEntities.stream()) .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber())) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index c820ee21..87452a6d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -115,9 +115,11 @@ rule "CBI.2.0: Don't redact genitive CBI_author" when $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied()) then - RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); - falsePositive.skip("CBI.2.0", "Genitive Author found"); - insert(falsePositive); + entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document) + .ifPresent(falsePositive -> { + falsePositive.skip("CBI.2.0", "Genitive Author found"); + insert(falsePositive); + }); end @@ -548,6 +550,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author" then nerEntities.streamEntitiesOfType("CBI_author") .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) + .filter(Optional::isPresent) + .map(Optional::get) .forEach(entity -> insert(entity)); end @@ -560,6 +564,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) + .filter(Optional::isPresent) + .map(Optional::get) .forEach(entity -> { entity.addEngine(Engine.NER); insert(entity); @@ -619,10 +625,12 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: RedactionEntity(matchesAnnotationId($id)) then $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.setRemoved(false); + $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); update($entityToForce); - retract($force); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); end @@ -649,7 +657,7 @@ rule "X.0.0: remove Entity contained by Entity of same type" $larger: RedactionEntity($type: type, $entityType: entityType) $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) then - $contained.removeFromGraph(); + $contained.remove(); retract($contained); end @@ -661,8 +669,8 @@ rule "X.1.0: merge intersecting Entities of same type" $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) then - $first.removeFromGraph(); - $second.removeFromGraph(); + $first.remove(); + $second.remove(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); retract($first); retract($second); @@ -679,7 +687,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.removeFromGraph(); + $entity.remove(); retract($entity) end @@ -691,7 +699,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -704,7 +712,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then $entity.addEngines($recommendation.getEngines()); - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -716,7 +724,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" $entity: RedactionEntity(entityType == EntityType.ENTITY) $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -729,7 +737,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.removeFromGraph(); + $lowerRank.remove(); retract($lowerRank); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index e4c9eb39..2d960529 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -132,9 +132,11 @@ rule "CBI.2.0: Don't redact genitive CBI_author" when $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied()) then - RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); - falsePositive.skip("CBI.2.0", "Genitive Author found"); - insert(falsePositive); + entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document) + .ifPresent(falsePositive -> { + falsePositive.skip("CBI.2.0", "Genitive Author found"); + insert(falsePositive); + }); end @@ -518,7 +520,7 @@ rule "CBI.13.0: Ignore CBI Address Recommendations" not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION) then - $entity.removeFromGraph(); + $entity.remove(); retract($entity) end @@ -651,11 +653,13 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") ) then - RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.removeFromGraph(); - retract($entityToExpand); - insert(expandedEntity); + entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") + .ifPresent(expandedEntity -> { + expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); + $entityToExpand.remove(); + retract($entityToExpand); + insert(expandedEntity); + }); end @@ -664,11 +668,13 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" when $entityToExpand: RedactionEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then - RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.removeFromGraph(); - retract($entityToExpand); - insert(expandedEntity); + entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") + .ifPresent(expandedEntity -> { + expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); + $entityToExpand.remove(); + retract($entityToExpand); + insert(expandedEntity); + }); end @@ -1139,10 +1145,12 @@ rule "PII.12.0: Expand PII entities with salutation prefix" when $entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then - RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - expandedEntity.addEngine(Engine.RULE); - insert(expandedEntity); + entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") + .ifPresent(expandedEntity -> { + expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); + expandedEntity.addEngine(Engine.RULE); + insert(expandedEntity); + }); end @@ -1287,6 +1295,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author" then nerEntities.streamEntitiesOfType("CBI_author") .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) + .filter(Optional::isPresent) + .map(Optional::get) .forEach(entity -> insert(entity)); end @@ -1299,6 +1309,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) + .filter(Optional::isPresent) + .map(Optional::get) .forEach(entity -> { entity.addEngine(Engine.NER); insert(entity); @@ -1315,6 +1327,8 @@ rule "AI.2.0: add all NER Entities of any type except CBI_author" nerEntities.getNerEntityList().stream() .filter(nerEntity -> !nerEntity.type().equals("CBI_author")) .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document)) + .filter(Optional::isPresent) + .map(Optional::get) .forEach(entity -> insert(entity)); end @@ -1371,10 +1385,12 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: RedactionEntity(matchesAnnotationId($id)) then $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.setRemoved(false); + $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); update($entityToForce); - retract($force); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); end @@ -1387,8 +1403,8 @@ rule "MAN.3.0: Apply image recategorization" then $imageToBeRecategorized.setImageType(ImageType.fromString($imageType)); update($imageToBeRecategorized); - retract($recategorization); update($imageToBeRecategorized.getParent()); + retract($recategorization); end @@ -1401,7 +1417,7 @@ rule "X.0.0: remove Entity contained by Entity of same type" $larger: RedactionEntity($type: type, $entityType: entityType) $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) then - $contained.removeFromGraph(); + $contained.remove(); retract($contained); end @@ -1413,8 +1429,8 @@ rule "X.1.0: merge intersecting Entities of same type" $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) then - $first.removeFromGraph(); - $second.removeFromGraph(); + $first.remove(); + $second.remove(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); retract($first); retract($second); @@ -1431,7 +1447,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.removeFromGraph(); + $entity.remove(); retract($entity) end @@ -1443,7 +1459,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -1456,7 +1472,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then $entity.addEngines($recommendation.getEngines()); - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -1468,7 +1484,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" $entity: RedactionEntity(entityType == EntityType.ENTITY) $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -1481,7 +1497,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.removeFromGraph(); + $lowerRank.remove(); retract($lowerRank); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index bd11ba30..00c9a4d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -374,6 +374,8 @@ rule "DOC.8.1: Performing Laboratory (Name)" nerEntities.streamEntitiesOfType("COUNTRY") .filter(nerEntity -> $section.getBoundary().contains(nerEntity.boundary())) .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section)) + .filter(Optional::isPresent) + .map(Optional::get) .forEach(entity -> { entity.apply("DOC.8.2", "Performing Laboratory found", "n-a"); insert(entity); @@ -588,8 +590,8 @@ rule "DOC.13.0: Clinical Signs" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "clinical_signs", EntityType.ENTITY, $section); - entity.apply("DOC.13.0", "Clinical Signs found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "clinical_signs", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.13.0", "Clinical Signs found", "n-a")); end @@ -618,8 +620,8 @@ rule "DOC.15.0: Mortality" $headline: Headline(containsString("Mortality") && !containsString("TABLE") && hasParagraphs()) FileAttribute(label == "OECD Number", value == "425") then - var entity = entityCreationService.byBoundary(Boundary.merge($headline.getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "mortality", EntityType.ENTITY, $headline.getParent()); - entity.apply("DOC.15.0", "Mortality found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "mortality", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.15.0", "Mortality found", "n-a")); end @@ -631,8 +633,8 @@ rule "DOC.17.0: Study Conclusion" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "study_conclusion", EntityType.ENTITY, $section); - entity.apply("DOC.17.0", "Study Conclusion found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "study_conclusion", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.17.0", "Study Conclusion found", "n-a")); end @@ -650,8 +652,8 @@ rule "DOC.18.0: Weight Behavior Changes" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "weight_behavior_changes", EntityType.ENTITY, $section); - entity.apply("DOC.18.0", "Weight behavior changes found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "weight_behavior_changes", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.18.0", "Weight behavior changes found", "n-a")); end @@ -669,8 +671,8 @@ rule "DOC.19.0: Necropsy findings" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "necropsy_findings", EntityType.ENTITY, $section); - entity.apply("DOC.19.0", "Necropsy section found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "necropsy_findings", EntityType.ENTITY) + .forEach( entity -> entity.apply("DOC.19.0", "Necropsy section found", "n-a")); end @@ -689,8 +691,8 @@ rule "DOC.22.0: Clinical observations" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "clinical_observations", EntityType.ENTITY, $section); - entity.apply("DOC.22.0", "Clinical observations section found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "clinical_observations", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.22.0", "Clinical observations section found", "n-a")); end @@ -746,8 +748,8 @@ rule "DOC.23.0: Bodyweight changes" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "bodyweight_changes", EntityType.ENTITY, $section); - entity.apply("DOC.23.0", "Bodyweight section found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "bodyweight_changes", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.23.0", "Bodyweight section found", "n-a")); end @@ -759,8 +761,8 @@ rule "DOC.24.0: Study Design" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "study_design", EntityType.ENTITY, $section); - entity.apply("DOC.24.0", "Study design section found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "study_design", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.24.0", "Study design section found", "n-a")); end @@ -781,8 +783,8 @@ rule "DOC.25.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "results_and_conclusion", EntityType.ENTITY, $section); - entity.apply("DOC.25.0", "Results and Conclusion found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "results_and_conclusion", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.25.0", "Results and Conclusion found", "n-a")); end @@ -816,8 +818,8 @@ rule "DOC.32.0: Preliminary Test Results (429)" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "preliminary_test_results", EntityType.ENTITY, $section); - entity.apply("DOC.32.0", "Preliminary Test Results found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "preliminary_test_results", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.32.0", "Preliminary Test Results found", "n-a")); end @@ -826,8 +828,8 @@ rule "DOC.33.0: Test Results (429)" FileAttribute(label == "OECD Number", value == "429") $section: Section((getHeadline().containsString("RESULTS AND DISCUSSION") || getHeadline().containsString("Estimation of the proliferative response of lymph node cells") || getHeadline().containsString("Results in the Main Experiment")) && hasParagraphs()) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "test_results", EntityType.ENTITY, $section); - entity.apply("DOC.33.0", "Test Results found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "test_results", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.33.0", "Test Results found", "n-a")); end @@ -962,8 +964,8 @@ rule "DOC.39.0: Dilution of the test substance" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "dilution", EntityType.ENTITY, $section); - entity.apply("DOC.39.0", "Dilution found.", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "dilution", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.39.0", "Dilution found.", "n-a")); end @@ -976,8 +978,8 @@ rule "DOC.40.0: Positive Control" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "positive_control", EntityType.ENTITY, $section); - entity.apply("DOC.40.0", "Positive control found.", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "positive_control", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.40.0", "Positive control found.", "n-a")); end @@ -986,8 +988,8 @@ rule "DOC.42.0: Mortality Statement" FileAttribute(label == "OECD Number", value == "402") $headline: Headline(containsString("Mortality") && !containsString("TABLE") && hasParagraphs()) then - var entity = entityCreationService.byBoundary(Boundary.merge($headline.getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "mortality_statement", EntityType.ENTITY, $headline.getParent()); - entity.apply("DOC.42.0", "Mortality Statement found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "mortality_statement", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.42.0", "Mortality Statement found", "n-a")); end @@ -1059,8 +1061,8 @@ rule "DOC.45.0: Doses (mg/kg bodyweight)" && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "doses_(mg_kg_bw)", EntityType.ENTITY, $section); - entity.apply("DOC.45.0", "Doses per bodyweight information found", "n-a"); + entityCreationService.bySemanticNodeParagraphsOnly($section, "doses_(mg_kg_bw)", EntityType.ENTITY) + .forEach(entity -> entity.apply("DOC.45.0", "Doses per bodyweight information found", "n-a")); end @@ -1106,11 +1108,16 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to rule "MAN.2.0: Apply force redaction" salience 128 when - ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) $entityToForce: RedactionEntity(matchesAnnotationId($id)) then $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.setRemoved(false); + $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + update($entityToForce); + $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 00a2d7ff..ddabbc57 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -469,11 +469,13 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") ) then - RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.removeFromGraph(); - retract($entityToExpand); - insert(expandedEntity); + entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") + .ifPresent(expandedEntity -> { + expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); + $entityToExpand.remove(); + retract($entityToExpand); + insert(expandedEntity); + }); end @@ -482,11 +484,13 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" when $entityToExpand: RedactionEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then - RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.removeFromGraph(); - retract($entityToExpand); - insert(expandedEntity); + entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") + .ifPresent(expandedEntity -> { + expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); + $entityToExpand.remove(); + retract($entityToExpand); + insert(expandedEntity); + }); end @@ -834,10 +838,12 @@ rule "PII.12.0: Expand PII entities with salutation prefix" when $entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then - RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - expandedEntity.addEngine(Engine.RULE); - insert(expandedEntity); + entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") + .ifPresent(expandedEntity -> { + expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); + expandedEntity.addEngine(Engine.RULE); + insert(expandedEntity); + }); end @@ -969,6 +975,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author" then nerEntities.streamEntitiesOfType("CBI_author") .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) + .filter(Optional::isPresent) + .map(Optional::get) .forEach(entity -> insert(entity)); end @@ -981,6 +989,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) + .filter(Optional::isPresent) + .map(Optional::get) .forEach(entity -> { entity.addEngine(Engine.NER); insert(entity); @@ -1040,10 +1050,12 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: RedactionEntity(matchesAnnotationId($id)) then $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.setRemoved(false); + $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); update($entityToForce); - retract($force); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); end @@ -1070,7 +1082,7 @@ rule "X.0.0: remove Entity contained by Entity of same type" $larger: RedactionEntity($type: type, $entityType: entityType) $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) then - $contained.removeFromGraph(); + $contained.remove(); retract($contained); end @@ -1082,8 +1094,8 @@ rule "X.1.0: merge intersecting Entities of same type" $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) then - $first.removeFromGraph(); - $second.removeFromGraph(); + $first.remove(); + $second.remove(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); retract($first); retract($second); @@ -1100,7 +1112,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.removeFromGraph(); + $entity.remove(); retract($entity) end @@ -1112,7 +1124,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -1125,7 +1137,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then $entity.addEngines($recommendation.getEngines()); - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -1137,7 +1149,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" $entity: RedactionEntity(entityType == EntityType.ENTITY) $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end @@ -1150,7 +1162,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.removeFromGraph(); + $lowerRank.remove(); retract($lowerRank); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 34382527..2c1a72c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -56,9 +56,11 @@ rule "add NER Entities of type CBI_author or CBI_address" when $nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address")) then - RedactionEntity redactionEntity = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document); - redactionEntity.addEngine(Engine.NER); - insert(redactionEntity); + entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document) + .ifPresent(redactionEntity -> { + redactionEntity.addEngine(Engine.NER); + insert(redactionEntity); + }); end // --------------------------------------- CBI rules ------------------------------------------------------------------- @@ -81,91 +83,126 @@ rule "Always redact PII" $cbiAuthor.apply("PII.0.0", "PII found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -// --------------------------------------- merging rules ------------------------------------------------------------------- +//------------------------------------ Entity merging rules ------------------------------------ -rule "remove Entity contained by Entity of same type" +// Rule unit: X.0 +rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when $larger: RedactionEntity($type: type, $entityType: entityType) $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) then - $contained.removeFromGraph(); + $contained.remove(); retract($contained); end -rule "merge intersecting Entities of same type" + +// Rule unit: X.1 +rule "X.1.0: merge intersecting Entities of same type" salience 64 when $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) then - $first.removeFromGraph(); - $second.removeFromGraph(); + $first.remove(); + $second.remove(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); retract($first); retract($second); insert(mergedEntity); + mergedEntity.getIntersectingNodes().forEach(node -> update(node)); end -rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" + +// Rule unit: X.2 +rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then - $entity.removeFromGraph(); + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.remove(); retract($entity) end -rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + +// Rule unit: X.3 +rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $recommendation.removeFromGraph(); + $recommendation.remove(); retract($recommendation); end -rule "remove Entity of type RECOMMENDATION when contained by ENTITY" - salience 64 + +// Rule unit: X.4 +rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" + salience 256 when $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $recommendation.removeFromGraph(); + $entity.addEngines($recommendation.getEngines()); + $recommendation.remove(); retract($recommendation); end -rule "remove Entity of lower rank, when equal boundaries and entityType" + +// Rule unit: X.5 +rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 256 + when + $entity: RedactionEntity(entityType == EntityType.ENTITY) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.remove(); + retract($recommendation); + end + + +// Rule unit: X.6 +rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" salience 32 when - $higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary) - $lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !applied) + $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY) + $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) then - $lowerRank.removeFromGraph(); + $lowerRank.getIntersectingNodes().forEach(node -> update(node)); + $lowerRank.remove(); retract($lowerRank); end -// --------------------------------------- FileAttribute Rules ------------------------------------------------------------------- -rule "remove duplicate FileAttributes" +//------------------------------------ File attributes rules ------------------------------------ + +// Rule unit: FA.1 +rule "FA.1.0: remove duplicate FileAttributes" salience 64 when - $first: FileAttribute($label: label, $value: value) - $second: FileAttribute(this != $first, label == $label, value == $value) + $fileAttribute: FileAttribute($label: label, $value: value) + $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value) then - retract($second); + retract($duplicate); end -// --------------------------------------- local dictionary search ------------------------------------------------------------------- -rule "run local dictionary search" +//------------------------------------ Local dictionary search rules ------------------------------------ + +// Rule unit: LDS.0 +rule "LDS.0.0: run local dictionary search" agenda-group "LOCAL_DICTIONARY_ADDS" salience -999 when DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) - .forEach(redactionEntity -> insert(redactionEntity)); + .forEach(entity -> { + entity.addEngine(Engine.RULE); + insert(entity); + }); end +