From f1090909cc256035875071124f0e77cdcbb2dc7d Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Tue, 29 Oct 2024 15:17:52 +0100 Subject: [PATCH] RED-10290: Improve SearchImplementation logic for dictionaries * cleanup --- .../v1/server/RedactionServiceSettings.java | 2 +- .../model/dictionary/DictionaryModel.java | 82 ------------------- .../service/DictionarySearchService.java | 40 --------- 3 files changed, 1 insertion(+), 123 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index 7b5a8439..39c9abae 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -26,7 +26,7 @@ public class RedactionServiceSettings { private boolean llmNerServiceEnabled; - private boolean priorityMode = false; + private boolean priorityMode; private long firstLevelDictionaryCacheMaximumSize = 1000; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java index 73423549..bffefbfb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java @@ -1,12 +1,9 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary; -import java.io.Serializable; import java.util.HashMap; import java.util.Locale; import java.util.Set; -import java.util.stream.Collectors; -import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; @@ -33,11 +30,6 @@ public class DictionaryModel implements Cloneable { private final Set falsePositives; private final Set falseRecommendations; - private SearchImplementation entriesSearch; - private SearchImplementation deletionEntriesSearch; - private SearchImplementation falsePositiveSearch; - private SearchImplementation falseRecommendationsSearch; - private final HashMap> localEntriesWithMatchedRules = new HashMap<>(); private SearchImplementation localSearch; @@ -91,72 +83,6 @@ public class DictionaryModel implements Cloneable { } - /** - * Returns the search implementation for non-deleted dictionary entries. - * - * @return The {@link SearchImplementation} for non-deleted dictionary entries. - */ - public SearchImplementation getEntriesSearch() { - - if (entriesSearch == null) { - this.entriesSearch = new SearchImplementation(this.entries.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .collect(Collectors.toList()), caseInsensitive); - } - return entriesSearch; - } - - - /** - * Returns the search implementation for deleted dictionary entries. - * - * @return The {@link SearchImplementation} for deleted dictionary entries. - */ - public SearchImplementation getDeletionEntriesSearch() { - - if (deletionEntriesSearch == null) { - this.deletionEntriesSearch = new SearchImplementation(this.entries.stream() - .filter(DictionaryEntry::isDeleted) - .map(DictionaryEntry::getValue) - .collect(Collectors.toList()), caseInsensitive); - } - return deletionEntriesSearch; - } - - - /** - * Returns the search implementation for non-deleted false positive entries. - * - * @return The {@link SearchImplementation} for non-deleted false positive entries. - */ - public SearchImplementation getFalsePositiveSearch() { - - if (falsePositiveSearch == null) { - this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .collect(Collectors.toList()), caseInsensitive); - } - return falsePositiveSearch; - } - - - /** - * Returns the search implementation for non-deleted false recommendation entries. - * - * @return The {@link SearchImplementation} for non-deleted false recommendation entries. - */ - public SearchImplementation getFalseRecommendationsSearch() { - - if (falseRecommendationsSearch == null) { - this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .collect(Collectors.toList()), caseInsensitive); - } - return falseRecommendationsSearch; - } /** @@ -173,20 +99,12 @@ public class DictionaryModel implements Cloneable { } - public SearchImplementation test() { - - return this.entriesSearch; - } @Override public DictionaryModel clone() { try { DictionaryModel cloned = (DictionaryModel) super.clone(); - cloned.entriesSearch = null; - cloned.deletionEntriesSearch = null; - cloned.falsePositiveSearch = null; - cloned.falseRecommendationsSearch = null; cloned.localSearch = null; return cloned; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index fb643f4f..61c12f5e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -62,44 +62,4 @@ public class DictionarySearchService { }); } - - @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries-old") - public void addDictionaryEntitiesOld(Dictionary dictionary, SemanticNode node) { - - dictionary.getDictionaryModels() - .forEach(model -> { - bySearchImplementationAsDictionary(model.getEntriesSearch(), - model.getType(), - model.isHint() ? EntityType.HINT : EntityType.ENTITY, - node, - model.isDossierDictionary()); - bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); - bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); - if (model.isDossierDictionary()) { - bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary()); - } - }); - } - - - public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation, - String type, - EntityType entityType, - SemanticNode node, - boolean isDossierDictionaryEntry) { - - Set engines = isDossierDictionaryEntry ? Set.of(Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); - searchImplementation.getBoundaries(node.getTextBlock()) - .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) - .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines) - .ifPresent(entity -> { - entity.setDictionaryEntry(true); - entity.setDossierDictionaryEntry(isDossierDictionaryEntry); - if (entityType.equals(EntityType.DICTIONARY_REMOVAL)) { - entity.ignore("DICT.0.0", "Ignore Dossier Dictionary Entity with DICTIONARY_REMOVAL entity type"); - } - })); - } - }