From f15db27a7c5f6ceb2ee1274263d89dfbd348f47f Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Tue, 29 Oct 2024 14:57:46 +0100 Subject: [PATCH] RED-10290: Improve SearchImplementation logic for dictionaries --- .../build.gradle.kts | 4 +- .../v1/server/RedactionServiceSettings.java | 2 +- .../server/model/dictionary/Dictionary.java | 273 +++++++++++++----- .../model/dictionary/DictionaryFactory.java | 107 +++---- ...rie.java => DictionaryIdentifierTrie.java} | 8 +- .../model/dictionary/DictionarySearch.java | 8 - .../DoubleArrayTrieDictionarySearch.java | 126 ++++++++ .../dictionary/HyperscanDictionarySearch.java | 172 ----------- .../dictionary/TrieDictionarySearch.java | 21 +- .../service/DictionarySearchService.java | 2 +- .../v1/server/service/DictionaryService.java | 2 +- .../v1/server/RedactionAcceptanceTest.java | 15 - .../graph/DictionaryPerformanceTest.java | 147 ---------- .../DictionarySearchImplementationsTest.java | 176 +++++++++++ 14 files changed, 560 insertions(+), 503 deletions(-) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/{DictionaryTrie.java => DictionaryIdentifierTrie.java} (84%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index f5ea1b3a..f205e939 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -61,7 +61,8 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") - implementation("org.ahocorasick:ahocorasick:0.6.3") + implementation("org.ahocorasick:ahocorasick:0.9.0") + implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2") implementation("org.javassist:javassist:3.29.2-GA") implementation("org.drools:drools-engine:${droolsVersion}") @@ -87,7 +88,6 @@ dependencies { implementation("org.reflections:reflections:0.10.2") implementation("com.opencsv:opencsv:5.9") - implementation("com.gliwka.hyperscan:hyperscan:5.4.11-3.0.0") implementation("com.joestelmach:natty:0.13") testImplementation(project(":rules-management")) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index 39c9abae..7b5a8439 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -26,7 +26,7 @@ public class RedactionServiceSettings { private boolean llmNerServiceEnabled; - private boolean priorityMode; + private boolean priorityMode = false; private long firstLevelDictionaryCacheMaximumSize = 1000; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index e18c13e0..1e4f040e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -2,14 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary; import static java.lang.String.format; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -31,28 +24,70 @@ import lombok.Getter; @Data public class Dictionary { - private final Map localAccessMap = new HashMap<>(); + private final Map> localAccessMap = new HashMap<>(); - @Getter private final DictionaryVersion version; private final DictionarySearch dictionarySearch; + public enum Level { + DOSSIER_TEMPLATE, + DOSSIER + } + Dictionary(List dictionaryModels, DictionaryVersion version, DictionarySearch dictionarySearch) { - dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm)); + dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), Map.of(getLevel(dm.isDossierDictionary()), dm))); this.version = version; this.dictionarySearch = dictionarySearch; } - public int getDictionaryRank(String type) { + private Level getLevel(boolean isDossierDictionary) { + + return isDossierDictionary ? Level.DOSSIER : Level.DOSSIER_TEMPLATE; + } + + + /** + * Determines the default level for a given type based on the levels present. + * If both levels are present, it defaults to {@code Level.DOSSIER}. + * + * @param type The type to determine the default level for. + * @return The default {@link Level} for the specified type. + * @throws NotFoundException If the type is not found in the dictionary. + */ + private Level getDefaultLevel(String type) { + + Map levelMap = localAccessMap.get(type); + if (levelMap == null || levelMap.isEmpty()) { + throw new NotFoundException("Type: " + type + " is not found"); + } + if (levelMap.containsKey(Level.DOSSIER)) { + return Level.DOSSIER; + } else { + // Use whatever level is present + return levelMap.keySet() + .iterator().next(); + } + } + + + public int getDictionaryRank(String type, Level level) { if (!localAccessMap.containsKey(type)) { return 0; } - return localAccessMap.get(type).getRank(); + DictionaryModel model = localAccessMap.get(type) + .get(level); + return model != null ? model.getRank() : 0; + } + + + public int getDictionaryRank(String type) { + + return getDictionaryRank(type, getDefaultLevel(type)); } @@ -72,6 +107,8 @@ public class Dictionary { return localAccessMap.values() .stream() + .flatMap(levelDictionaryModelMap -> levelDictionaryModelMap.values() + .stream()) .toList(); } @@ -83,56 +120,144 @@ public class Dictionary { /** - * Retrieves the {@link DictionaryModel} of a specified type. + * Retrieves the {@link DictionaryModel} of a specified type and level. * - * @param type The type of dictionary model to retrieve. - * @return The {@link DictionaryModel} of the specified type. - * @throws NotFoundException If the specified type is not found in the dictionary. + * @param type The type of dictionary model to retrieve. + * @param level The level of the dictionary model to retrieve. + * @return The {@link DictionaryModel} of the specified type and level. + * @throws NotFoundException If the specified type or level is not found in the dictionary. */ - public DictionaryModel getType(String type) { + public DictionaryModel getType(String type, Level level) { - DictionaryModel model = localAccessMap.get(type); - if (model == null) { - throw new NotFoundException("Type: " + type + " is not found"); + Map levelMap = localAccessMap.get(type); + if (levelMap == null || !levelMap.containsKey(level)) { + throw new NotFoundException("Type: " + type + " with level: " + level + " is not found"); } - return model; + return levelMap.get(level); } /** - * Checks if the dictionary of a specific type is considered a hint. + * Retrieves the {@link DictionaryModel} of a specified type at the default level. + * + * @param type The type of dictionary model to retrieve. + * @return The {@link DictionaryModel} of the specified type at the default level. + * @throws NotFoundException If the specified type is not found in the dictionary. + */ + public DictionaryModel getType(String type) { + + return getType(type, getDefaultLevel(type)); + } + + + /** + * Checks if the dictionary of a specific type and level is considered a hint. + * + * @param type The type of dictionary to check. + * @param level The level of the dictionary to check. + * @return true if the dictionary model is marked as a hint, false otherwise. + */ + public boolean isHint(String type, Level level) { + + DictionaryModel model = localAccessMap.get(type) + .get(level); + return model != null && model.isHint(); + } + + + /** + * Checks if the dictionary of a specific type is considered a hint at the default level. * * @param type The type of dictionary to check. * @return true if the dictionary model is marked as a hint, false otherwise. */ public boolean isHint(String type) { - DictionaryModel model = localAccessMap.get(type); - if (model != null) { - return model.isHint(); - } - return false; + return isHint(type, getDefaultLevel(type)); } /** - * Checks if the dictionary of a specific type is case-insensitive. + * Checks if the dictionary of a specific type and level is case-insensitive. + * + * @param type The type of dictionary to check. + * @param level The level of the dictionary to check. + * @return true if the dictionary is case-insensitive, false otherwise. + */ + public boolean isCaseInsensitiveDictionary(String type, Level level) { + + DictionaryModel dictionaryModel = localAccessMap.get(type) + .get(level); + return dictionaryModel != null && dictionaryModel.isCaseInsensitive(); + } + + + /** + * Checks if the dictionary of a specific type is case-insensitive at the default level. * * @param type The type of dictionary to check. * @return true if the dictionary is case-insensitive, false otherwise. */ public boolean isCaseInsensitiveDictionary(String type) { - DictionaryModel dictionaryModel = localAccessMap.get(type); - if (dictionaryModel != null) { - return dictionaryModel.isCaseInsensitive(); - } - return false; + return isCaseInsensitiveDictionary(type, getDefaultLevel(type)); } /** - * Adds a local dictionary entry of a specific type. + * Adds a local dictionary entry of a specific type and level. + * + * @param type The type of dictionary to add the entry to. + * @param value The value of the entry. + * @param matchedRules A collection of {@link MatchedRule} associated with the entry. + * @param alsoAddLastname Indicates whether to also add the lastname separately as an entry. + * @param level The level of the dictionary where the entry should be added. + * @throws IllegalArgumentException If the specified type does not exist within the dictionary, if the type + * does not have any local entries defined, or if the provided value is + * blank. This ensures that only valid, non-empty entries + * are added to the dictionary. + */ + private void addLocalDictionaryEntry(String type, String value, Collection matchedRules, boolean alsoAddLastname, Level level) { + + if (value.isBlank()) { + return; + } + Map levelMap = localAccessMap.get(type); + if (levelMap == null || !levelMap.containsKey(level)) { + throw new IllegalArgumentException(format("DictionaryModel of type %s with level %s does not exist", type, level)); + } + DictionaryModel dictionaryModel = levelMap.get(level); + if (dictionaryModel.getLocalEntriesWithMatchedRules() == null) { + throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type)); + } + if (StringUtils.isEmpty(value)) { + throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value)); + } + boolean isCaseInsensitive = dictionaryModel.isCaseInsensitive(); + Set matchedRulesSet = new HashSet<>(matchedRules); + + String cleanedValue = value; + if (isCaseInsensitive) { + cleanedValue = cleanedValue.toLowerCase(Locale.US); + } + dictionaryModel.getLocalEntriesWithMatchedRules() + .merge(cleanedValue.trim(), + matchedRulesSet, + (set1, set2) -> Stream.concat(set1.stream(), set2.stream()) + .collect(Collectors.toSet())); + if (alsoAddLastname) { + String lastname = cleanedValue.split(" ")[0]; + dictionaryModel.getLocalEntriesWithMatchedRules() + .merge(lastname, + matchedRulesSet, + (set1, set2) -> Stream.concat(set1.stream(), set2.stream()) + .collect(Collectors.toSet())); + } + } + + + /** + * Adds a local dictionary entry of a specific type at the default level. * * @param type The type of dictionary to add the entry to. * @param value The value of the entry. @@ -145,40 +270,7 @@ public class Dictionary { */ private void addLocalDictionaryEntry(String type, String value, Collection matchedRules, boolean alsoAddLastname) { - if (value.isBlank()) { - return; - } - if (localAccessMap.get(type) == null) { - throw new IllegalArgumentException(format("DictionaryModel of type %s does not exist", type)); - } - if (localAccessMap.get(type).getLocalEntriesWithMatchedRules() == null) { - throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type)); - } - if (StringUtils.isEmpty(value)) { - throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value)); - } - boolean isCaseInsensitive = localAccessMap.get(type).isCaseInsensitive(); - Set matchedRulesSet = new HashSet<>(matchedRules); - - String cleanedValue = value; - if (isCaseInsensitive) { - cleanedValue = cleanedValue.toLowerCase(Locale.US); - } - localAccessMap.get(type) - .getLocalEntriesWithMatchedRules() - .merge(cleanedValue.trim(), - matchedRulesSet, - (set1, set2) -> Stream.concat(set1.stream(), set2.stream()) - .collect(Collectors.toSet())); - if (alsoAddLastname) { - String lastname = cleanedValue.split(" ")[0]; - localAccessMap.get(type) - .getLocalEntriesWithMatchedRules() - .merge(lastname, - matchedRulesSet, - (set1, set2) -> Stream.concat(set1.stream(), set2.stream()) - .collect(Collectors.toSet())); - } + addLocalDictionaryEntry(type, value, matchedRules, alsoAddLastname, getDefaultLevel(type)); } @@ -186,10 +278,22 @@ public class Dictionary { * Recommends a text entity for inclusion in every dictionary model without separating the last name. * * @param textEntity The {@link TextEntity} to be recommended. + * @param level The level of the dictionary where the recommendation should be added. + */ + public void recommendEverywhere(TextEntity textEntity, Level level) { + + addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false, level); + } + + + /** + * Recommends a text entity for inclusion in every dictionary model without separating the last name at the default level. + * + * @param textEntity The {@link TextEntity} to be recommended. */ public void recommendEverywhere(TextEntity textEntity) { - addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false); + recommendEverywhere(textEntity, getDefaultLevel(textEntity.type())); } @@ -197,10 +301,22 @@ public class Dictionary { * Recommends a text entity for inclusion in every dictionary model with the last name added separately. * * @param textEntity The {@link TextEntity} to be recommended. + * @param level The level of the dictionary where the recommendation should be added. + */ + public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity, Level level) { + + addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true, level); + } + + + /** + * Recommends a text entity for inclusion in every dictionary model with the last name added separately at the default level. + * + * @param textEntity The {@link TextEntity} to be recommended. */ public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) { - addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true); + recommendEverywhereWithLastNameSeparately(textEntity, getDefaultLevel(textEntity.type())); } @@ -208,11 +324,22 @@ public class Dictionary { * Adds multiple author names contained within a text entity as recommendations in the dictionary. * * @param textEntity The {@link TextEntity} containing author names to be added. + * @param level The level of the dictionary where the recommendations should be added. + */ + public void addMultipleAuthorsAsRecommendation(TextEntity textEntity, Level level) { + + splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true, level)); + } + + + /** + * Adds multiple author names contained within a text entity as recommendations in the dictionary at the default level. + * + * @param textEntity The {@link TextEntity} containing author names to be added. */ public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) { - splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true)); - + addMultipleAuthorsAsRecommendation(textEntity, getDefaultLevel(textEntity.type())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java index 16efc3bb..decc2d99 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java @@ -1,13 +1,15 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.springframework.stereotype.Service; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; -import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import lombok.RequiredArgsConstructor; @@ -17,24 +19,12 @@ import lombok.SneakyThrows; @RequiredArgsConstructor public class DictionaryFactory { - private final RedactionServiceSettings settings; - @SneakyThrows public Dictionary create(List dictionaryModels, DictionaryVersion dictionaryVersion) { - // Generate the combined dictionary values map Map> combinedMap = generateCombinedDictionaryValuesMap(dictionaryModels); - - // Determine which DictionarySearch implementation to use - DictionarySearch dictionarySearch; - if (settings.isPriorityMode()) { - dictionarySearch = new HyperscanDictionarySearch(combinedMap); - } else { - dictionarySearch = new TrieDictionarySearch(combinedMap); - } - - // Instantiate and return the Dictionary + DictionarySearch dictionarySearch = new DoubleArrayTrieDictionarySearch(combinedMap); return new Dictionary(dictionaryModels, dictionaryVersion, dictionarySearch); } @@ -42,62 +32,45 @@ public class DictionaryFactory { private Map> generateCombinedDictionaryValuesMap(List dictionaryModels) { Map> combinedValuesMap = new HashMap<>(); - for (DictionaryModel model : dictionaryModels) { - - DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive()); - DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(), - EntityType.DICTIONARY_REMOVAL, - model.isDossierDictionary(), - !model.isCaseInsensitive()); - DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(), - EntityType.FALSE_POSITIVE, - model.isDossierDictionary(), - !model.isCaseInsensitive()); - DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(), - EntityType.FALSE_RECOMMENDATION, - model.isDossierDictionary(), - !model.isCaseInsensitive()); - - List entryValues = model.getEntries() - .stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .toList(); - - List deletedEntryValues = model.getEntries() - .stream() - .filter(DictionaryEntry::isDeleted) - .map(DictionaryEntry::getValue) - .toList(); - - List falsePositiveValues = model.getFalsePositives() - .stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .toList(); - - List falseRecommendationValues = model.getFalseRecommendations() - .stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .toList(); - - if (!entryValues.isEmpty()) { - combinedValuesMap.put(entriesIdentifier, entryValues); - } - if (!deletedEntryValues.isEmpty()) { - combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues); - } - if (!falsePositiveValues.isEmpty()) { - combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues); - } - if (!falseRecommendationValues.isEmpty()) { - combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues); - } + addDictionaryEntries(combinedValuesMap, model); } - return combinedValuesMap; } + + private void addDictionaryEntries(Map> combinedValuesMap, DictionaryModel model) { + + addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.ENTITY), filterValues(model.getEntries(), false)); + addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.FALSE_POSITIVE), filterValues(model.getFalsePositives(), false)); + addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.FALSE_RECOMMENDATION), filterValues(model.getFalseRecommendations(), false)); + + if (model.isDossierDictionary()) { + addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.DICTIONARY_REMOVAL), filterValues(model.getEntries(), true)); + } + } + + + private DictionaryIdentifier createIdentifier(DictionaryModel model, EntityType entityType) { + + return new DictionaryIdentifier(model.getType(), entityType, model.isDossierDictionary(), !model.isCaseInsensitive()); + } + + + private List filterValues(Set entries, boolean isDeleted) { + + return entries.stream() + .filter(entry -> entry.isDeleted() == isDeleted) + .map(DictionaryEntry::getValue) + .toList(); + } + + + private void addValuesToMap(Map> map, DictionaryIdentifier key, List values) { + + if (!values.isEmpty()) { + map.computeIfAbsent(key, k -> new ArrayList<>()).addAll(values); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryTrie.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifierTrie.java similarity index 84% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryTrie.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifierTrie.java index a6a5c33e..b2a8c291 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryTrie.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifierTrie.java @@ -5,10 +5,10 @@ import org.ahocorasick.trie.PayloadTrie; import java.util.Collection; -public class DictionaryTrie { +public class DictionaryIdentifierTrie { private final PayloadTrie trie; - private DictionaryTrie(PayloadTrie trie) { + private DictionaryIdentifierTrie(PayloadTrie trie) { this.trie = trie; } @@ -36,8 +36,8 @@ public class DictionaryTrie { return this; } - public DictionaryTrie build() { - return new DictionaryTrie(builder.build()); + public DictionaryIdentifierTrie build() { + return new DictionaryIdentifierTrie(builder.build()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java index e96ef54e..594ebbe9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java @@ -10,14 +10,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl */ public interface DictionarySearch { - /** - * Checks if at least one dictionary entry matches the given text. - * - * @param text The text to search within. - * @return True if at least one match is found, otherwise false. - */ - boolean atLeastOneMatches(String text); - /** * Retrieves a list of match boundaries within the given text. * diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java new file mode 100644 index 00000000..42a2bfee --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java @@ -0,0 +1,126 @@ +package com.iqser.red.service.redaction.v1.server.model.dictionary; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Stream; + +import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; + +public class DoubleArrayTrieDictionarySearch implements DictionarySearch { + + private final AhoCorasickDoubleArrayTrie> trie; + + + public DoubleArrayTrieDictionarySearch(Map> dictionaryValues) { + + trie = new AhoCorasickDoubleArrayTrie<>(); + trie.build(computeStringIdentifiersMap(dictionaryValues)); + } + + + private static Map> computeStringIdentifiersMap(Map> dictionaryValues) { + + Map> stringToIdentifiersMap = new HashMap<>(); + for (Map.Entry> entry : dictionaryValues.entrySet()) { + DictionaryIdentifier identifier = entry.getKey(); + List values = entry.getValue(); + for (String value : values) { + DictionaryIdentifierWithKeyword idWithKeyword = new DictionaryIdentifierWithKeyword(identifier, value); + stringToIdentifiersMap.computeIfAbsent(value.toLowerCase(Locale.ROOT), k -> new ArrayList<>()).add(idWithKeyword); + } + } + return stringToIdentifiersMap; + } + + + @Override + public Stream getBoundaries(CharSequence text) { + + String lowerText = text.toString().toLowerCase(Locale.ROOT); + return getMatchTextRangeStream(text, lowerText, 0); + } + + + @Override + public Stream getBoundaries(TextBlock textBlock) { + + return getBoundaries(textBlock, textBlock.getTextRange()); + } + + + @Override + public Stream getBoundaries(CharSequence text, TextRange region) { + + String subText = text.subSequence(region.start(), region.end()).toString(); + String lowerSubText = subText.toLowerCase(Locale.ROOT); + return getMatchTextRangeStream(text, lowerSubText, region.start()); + } + + + @Override + public Stream getMatches(String text) { + + List matches = new ArrayList<>(); + String lowerText = text.toLowerCase(Locale.ROOT); + List>> hits = trie.parseText(lowerText); + + for (AhoCorasickDoubleArrayTrie.Hit> hit : hits) { + String matchedText = text.substring(hit.begin, hit.end); + List idWithKeywords = hit.value; + + for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) { + MatchPosition matchPosition = new MatchPosition(idkw.identifier, hit.begin, hit.end); + if (idkw.identifier.caseSensitive()) { + if (matchedText.equals(idkw.keyword)) { + matches.add(matchPosition); + } + } else { + matches.add(matchPosition); + } + } + } + return matches.stream(); + } + + + private Stream getMatchTextRangeStream(CharSequence text, String lowerText, int offset) { + + List matches = new ArrayList<>(); + List>> hits = trie.parseText(lowerText); + + for (AhoCorasickDoubleArrayTrie.Hit> hit : hits) { + addMatchesForHit(text, matches, hit, offset); + } + return matches.stream(); + } + + + private void addMatchesForHit(CharSequence text, List matches, AhoCorasickDoubleArrayTrie.Hit> hit, int offset) { + + int start = hit.begin + offset; + int end = hit.end + offset; + String matchedText = text.subSequence(start, end).toString(); + List idWithKeywords = hit.value; + + for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) { + if (idkw.identifier.caseSensitive()) { + if (matchedText.equals(idkw.keyword)) { + matches.add(new MatchTextRange(idkw.identifier, new TextRange(start, end))); + } + } else { + matches.add(new MatchTextRange(idkw.identifier, new TextRange(start, end))); + } + } + } + + + private record DictionaryIdentifierWithKeyword(DictionaryIdentifier identifier, String keyword) { + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java deleted file mode 100644 index 7c692d7d..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java +++ /dev/null @@ -1,172 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.model.dictionary; - -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import com.gliwka.hyperscan.wrapper.CompileErrorException; -import com.gliwka.hyperscan.wrapper.Database; -import com.gliwka.hyperscan.wrapper.Expression; -import com.gliwka.hyperscan.wrapper.ExpressionFlag; -import com.gliwka.hyperscan.wrapper.Match; -import com.gliwka.hyperscan.wrapper.Scanner; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; -import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; - -public class HyperscanDictionarySearch implements DictionarySearch { - - private final Map patternIdMap = new ConcurrentHashMap<>(); - private AtomicInteger patternIdCounter = new AtomicInteger(0); - private Database database; - - - public HyperscanDictionarySearch(Map> dictionaryValues) throws CompileErrorException { - - Map> entries = new HashMap<>(dictionaryValues); - this.database = compileDatabase(entries, this.patternIdMap); - } - - - private Database compileDatabase(Map> entries, Map patternIdMap) throws CompileErrorException { - - if (entries.isEmpty()) { - return null; - } - - LinkedList expressions = new LinkedList<>(); - - for (Map.Entry> entry : entries.entrySet()) { - DictionaryIdentifier identifier = entry.getKey(); - for (String value : entry.getValue()) { - int patternId = patternIdCounter.incrementAndGet(); - patternIdMap.put(patternId, identifier); - - EnumSet flags = EnumSet.noneOf(ExpressionFlag.class); - if (!identifier.caseSensitive()) { - flags.add(ExpressionFlag.CASELESS); - } - // escape regex characters as we only use literals - expressions.add(new Expression(Pattern.quote(value), flags, patternId)); - } - } - - // Build the database - return Database.compile(expressions); - } - - - @Override - public boolean atLeastOneMatches(String text) { - - if (this.database == null) { - return false; - } - - try (Scanner scanner = new Scanner()) { - scanner.allocScratch(this.database); - List matches = scanner.scan(this.database, text); - return !matches.isEmpty(); - } - } - - - @Override - public Stream getBoundaries(CharSequence text) { - - List boundaries = new ArrayList<>(); - - if (this.database == null) { - return Stream.empty(); - } - - try (Scanner scanner = new Scanner()) { - scanner.allocScratch(this.database); - List matches = scanner.scan(this.database, text.toString()); - for (Match match : matches) { - int start = (int) match.getStartPosition(); - int end = (int) match.getEndPosition(); - int patternId = match.getMatchedExpression().getId(); - DictionaryIdentifier identifier = patternIdMap.get(patternId); - boundaries.add(new MatchTextRange(identifier, new TextRange(start, end))); - } - } - - return boundaries.stream(); - } - - - @Override - public Stream getBoundaries(TextBlock textBlock) { - - return getBoundaries(textBlock, textBlock.getTextRange()); - } - - - @Override - public Stream getBoundaries(CharSequence text, TextRange region) { - - List boundaries = new ArrayList<>(); - - if (this.database == null) { - return Stream.empty(); - } - - CharSequence subText = text.subSequence(region.start(), region.end()); - - try (Scanner scanner = new Scanner()) { - scanner.allocScratch(this.database); - List matches = scanner.scan(this.database, subText.toString()); - for (Match match : matches) { - int start = (int) (match.getStartPosition() + region.start()); - int end = (int) (match.getEndPosition() + region.start()); - int patternId = match.getMatchedExpression().getId(); - DictionaryIdentifier identifier = patternIdMap.get(patternId); - boundaries.add(new MatchTextRange(identifier, new TextRange(start, end))); - } - } - - return boundaries.stream(); - } - - - @Override - public Stream getMatches(String text) { - - List matchPositions = new ArrayList<>(); - - if (this.database == null) { - return Stream.empty(); - } - - try (Scanner scanner = new Scanner()) { - scanner.allocScratch(this.database); - List matches = scanner.scan(this.database, text); - for (Match match : matches) { - int start = (int) match.getStartPosition(); - int end = (int) match.getEndPosition(); - int patternId = match.getMatchedExpression().getId(); - DictionaryIdentifier identifier = patternIdMap.get(patternId); - matchPositions.add(new MatchPosition(identifier, start, end)); - } - } - - return matchPositions.stream(); - } - - - public void close() { - - if (this.database != null) { - this.database.close(); - this.database = null; - } - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java index 39c56e53..4c646885 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; @@ -12,15 +11,14 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl public class TrieDictionarySearch implements DictionarySearch { - private final Map> caseSensitiveEntries = new ConcurrentHashMap<>(); - private final Map> caseInsensitiveEntries = new ConcurrentHashMap<>(); - private final DictionaryTrie caseSensitiveTrie; - private final DictionaryTrie caseInsensitiveTrie; + private final Map> caseSensitiveEntries = new HashMap<>(); + private final Map> caseInsensitiveEntries = new HashMap<>(); + private final DictionaryIdentifierTrie caseSensitiveTrie; + private final DictionaryIdentifierTrie caseInsensitiveTrie; public TrieDictionarySearch(Map> dictionaryValues) { - System.out.println("TEST ---> DictionarySearchImplementation CTOR"); for (Map.Entry> entry : dictionaryValues.entrySet()) { DictionaryIdentifier identifier = entry.getKey(); List values = entry.getValue(); @@ -36,12 +34,12 @@ public class TrieDictionarySearch implements DictionarySearch { } - private DictionaryTrie createTrie(Map> entries, boolean ignoreCase) { + private DictionaryIdentifierTrie createTrie(Map> entries, boolean ignoreCase) { if (entries.isEmpty()) { return null; } - DictionaryTrie.Builder builder = new DictionaryTrie.Builder(); + DictionaryIdentifierTrie.Builder builder = new DictionaryIdentifierTrie.Builder(); if (ignoreCase) { builder.ignoreCase(); } @@ -54,7 +52,6 @@ public class TrieDictionarySearch implements DictionarySearch { } - @Override public boolean atLeastOneMatches(String text) { if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) { @@ -101,7 +98,7 @@ public class TrieDictionarySearch implements DictionarySearch { } - private void addMatchTextRangesForTrie(Map> entries, DictionaryTrie trie, List matches, CharSequence text) { + private void addMatchTextRangesForTrie(Map> entries, DictionaryIdentifierTrie trie, List matches, CharSequence text) { if (!entries.isEmpty() && trie != null) { matches.addAll(trie.parseText(text) @@ -116,7 +113,7 @@ public class TrieDictionarySearch implements DictionarySearch { TextRange region, List matches, Map> entries, - DictionaryTrie trie) { + DictionaryIdentifierTrie trie) { if (!entries.isEmpty() && trie != null) { CharSequence subSequence = text.subSequence(region.start(), region.end()); @@ -128,7 +125,7 @@ public class TrieDictionarySearch implements DictionarySearch { } - private void addMatchPositionsForTrie(Map> entries, DictionaryTrie trie, List matches, String text) { + private void addMatchPositionsForTrie(Map> entries, DictionaryIdentifierTrie trie, List matches, String text) { if (!entries.isEmpty() && trie != null) { matches.addAll(trie.parseText(text) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index fb643f4f..48226c8c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -31,7 +31,7 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, List semanticNodes) { - semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node)); + semanticNodes.parallelStream().forEach(node -> addDictionaryEntities(dictionary, node)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index 91641adb..291250b1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -223,7 +223,7 @@ public class DictionaryService { combinedEntries, combinedFalsePositives, combinedFalseRecommendations, - type.isDossierDictionaryOnly()); + dossierId != null); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 1d9c920d..65d96ce9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -58,21 +58,6 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { @Autowired DictionaryCacheService dictionaryCacheService; - @Configuration - @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) - @Import(LayoutParsingServiceProcessorConfiguration.class) - @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) - public static class RedactionIntegrationTestConfiguration { - - @Bean - @Primary - public StorageService inmemoryStorage() { - - return new FileSystemBackedStorageService(ObjectMapperFactory.create()); - } - - } - @BeforeEach public void stubClients() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java deleted file mode 100644 index f91c9758..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java +++ /dev/null @@ -1,147 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import static org.springframework.test.util.AssertionErrors.assertEquals; - -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch; -import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch; -import com.iqser.red.service.redaction.v1.server.model.dictionary.HyperscanDictionarySearch; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; -import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.gliwka.hyperscan.wrapper.CompileErrorException; - -@Disabled -public class DictionaryPerformanceTest { - - private static final int LARGE_DICTIONARY_SIZE = 5_000; - private static final int LARGE_TEXT_REPETITIONS = 50_000; - private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " - + "Entity_1 match text. Recommendation_1 also here. Random text continues. "; - - @Test - public void testLargeFileWithLargeDictionaries() { - - Map> dictionaryValues = new HashMap<>(); - - // Generate dictionary values with specific terms for matching - IntStream.range(0, 6) - .forEach(i -> { - EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION; - boolean caseSensitive = i % 2 == 0; - - DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, - caseSensitive); - List dictionary = generateLargeDictionary(); - - // Add specific terms that are included in the large text for matches - if (i == 0) { - dictionary.add("Entity_1"); - } - if (i == 1) { - dictionary.add("recommendation_1"); - } - dictionaryValues.put(identifier, dictionary); - }); - - // Measure construction time for DictionarySearchImplementation - long dictionaryTrieConstructionStart = System.currentTimeMillis(); - TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(dictionaryValues); - long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart; - - // Measure construction time for SearchImplementations - long searchTrieConstructionStart = System.currentTimeMillis(); - List searchImplementations = dictionaryValues.entrySet().stream() - .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive())).toList(); - long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart; - - // Measure construction time for HyperSearchImplementation - long hyperSearchConstructionStart = System.currentTimeMillis(); - HyperscanDictionarySearch hyperSearchImpl = null; - try { - hyperSearchImpl = new HyperscanDictionarySearch(dictionaryValues); - } catch (CompileErrorException e) { - assert false : "Failed to compile HyperSearchImplementation: " + e.getMessage(); - } - long hyperSearchConstructionDuration = System.currentTimeMillis() - hyperSearchConstructionStart; - - String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS); - - // Measure search time for DictionarySearchImplementation - long dictionarySearchStart = System.currentTimeMillis(); - List dictionaryMatches = dictionarySearchImpl - .getBoundariesAsList(largeText); - long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart; - - // Measure search time for SearchImplementations - long searchImplStart = System.currentTimeMillis(); - List searchMatches = new ArrayList<>(); - for (SearchImplementation searchImpl : searchImplementations) { - searchMatches.addAll(searchImpl.getBoundaries(largeText)); - } - long searchImplDuration = System.currentTimeMillis() - searchImplStart; - - // Measure search time for HyperSearchImplementation - long hyperSearchStart = System.currentTimeMillis(); - List hyperMatches = hyperSearchImpl.getMatchesAsList(largeText); - long hyperSearchDuration = System.currentTimeMillis() - hyperSearchStart; - - System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration); - System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration, - dictionaryMatches.size()); - System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration); - System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration, - searchMatches.size()); - System.out.printf("HyperSearchImplementation construction took %d ms\n", hyperSearchConstructionDuration); - System.out.printf("HyperSearchImplementation search took %d ms and found %d matches\n", hyperSearchDuration, - hyperMatches.size()); - - assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() && !hyperMatches.isEmpty() - : "All implementations should find entities."; - - // Ensure all implementations found the same number of matches - assertEquals("DictionarySearchImplementation and SearchImplementations should find the same number of matches", - dictionaryMatches.size(), searchMatches.size()); - assertEquals("DictionarySearchImplementation and HyperSearchImplementation should find the same number of matches", - dictionaryMatches.size(), hyperMatches.size()); - - // Close the HyperSearchImplementation to release resources - hyperSearchImpl.close(); - } - - private List generateLargeDictionary() { - return IntStream.range(0, LARGE_DICTIONARY_SIZE) - .mapToObj(i -> RandomStringGenerator.generateRandomString()) - .collect(Collectors.toList()); - } - - static final class RandomStringGenerator { - - private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - private static final int STRING_LENGTH = 50; - private static final SecureRandom RANDOM = new SecureRandom(); - - public static String generateRandomString() { - - StringBuilder sb = new StringBuilder(STRING_LENGTH); - for (int i = 0; i < STRING_LENGTH; i++) { - int index = RANDOM.nextInt(CHARACTERS.length()); - sb.append(CHARACTERS.charAt(index)); - } - return sb.toString(); - } - - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java new file mode 100644 index 00000000..f40ad83a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java @@ -0,0 +1,176 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; + +import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DoubleArrayTrieDictionarySearch; +import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; + +public class DictionarySearchImplementationsTest { + + private static final int LARGE_DICTIONARY_SIZE = 4_000; + private static final int LARGE_TEXT_REPETITIONS = 500_000; + private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " + + "Entity_1 match text. Recommendation_1 also here. Random text continues. "; + + + @Test +//@Disabled + public void performanceTest() { + + Map> dictionaryValues = new HashMap<>(); + + // Generate dictionary values with specific terms for matching + IntStream.range(0, 6) + .forEach(i -> { + EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION; + boolean caseSensitive = i % 2 == 0; + + DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive); + List dictionary = generateLargeDictionary(); + + // Add specific terms that are included in the large text for matches + if (i == 0) { + dictionary.add("Entity_1"); + } + if (i == 1) { + dictionary.add("recommendation_1"); + } + dictionaryValues.put(identifier, dictionary); + }); + + // Measure construction time for TrieDictionarySearch + long trieDictionaryConstructionStart = System.currentTimeMillis(); + TrieDictionarySearch trieDictionarySearchImpl = new TrieDictionarySearch(dictionaryValues); + long trieDictionaryConstructionDuration = System.currentTimeMillis() - trieDictionaryConstructionStart; + + // Measure construction time for SearchImplementations + long searchTrieConstructionStart = System.currentTimeMillis(); + List searchImplementations = dictionaryValues.entrySet() + .stream() + .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive())) + .toList(); + long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart; + + // Measure construction time for DoubleArrayTrieDictionarySearch + long doubleArrayTrieConstructionStart = System.currentTimeMillis(); + DoubleArrayTrieDictionarySearch doubleArrayTrieSearchImpl = new DoubleArrayTrieDictionarySearch(dictionaryValues); + long doubleArrayTrieConstructionDuration = System.currentTimeMillis() - doubleArrayTrieConstructionStart; + + String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS); + + // Measure search time for TrieDictionarySearch + long trieDictionarySearchStart = System.currentTimeMillis(); + List trieDictionaryMatches = trieDictionarySearchImpl.getBoundariesAsList(largeText); + long trieDictionarySearchDuration = System.currentTimeMillis() - trieDictionarySearchStart; + + // Measure search time for SearchImplementations + long searchImplStart = System.currentTimeMillis(); + List searchMatches = new ArrayList<>(); + for (SearchImplementation searchImpl : searchImplementations) { + searchMatches.addAll(searchImpl.getBoundaries(largeText)); + } + long searchImplDuration = System.currentTimeMillis() - searchImplStart; + + // Measure search time for DoubleArrayTrieDictionarySearch + long doubleArrayTrieSearchStart = System.currentTimeMillis(); + List doubleArrayTrieMatches = doubleArrayTrieSearchImpl.getBoundariesAsList(largeText); + long doubleArrayTrieSearchDuration = System.currentTimeMillis() - doubleArrayTrieSearchStart; + + // Output the performance results + System.out.printf("TrieDictionarySearch construction took %d ms\n", trieDictionaryConstructionDuration); + System.out.printf("TrieDictionarySearch search took %d ms and found %d matches\n", trieDictionarySearchDuration, trieDictionaryMatches.size()); + System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration); + System.out.printf("Combined SearchImplementation search took %d ms and found %d matches\n", searchImplDuration, searchMatches.size()); + System.out.printf("DoubleArrayTrieDictionarySearch construction took %d ms\n", doubleArrayTrieConstructionDuration); + System.out.printf("DoubleArrayTrieDictionarySearch search took %d ms and found %d matches\n", doubleArrayTrieSearchDuration, doubleArrayTrieMatches.size()); + + // Assert that all implementations found matches + assert !trieDictionaryMatches.isEmpty() + && !searchMatches.isEmpty() + && !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities."; + + // Ensure all implementations found the same number of matches + assertEquals(trieDictionaryMatches.size(), searchMatches.size(), "Mismatch between TrieDictionarySearch and SearchImplementations"); + assertEquals(trieDictionaryMatches.size(), doubleArrayTrieMatches.size(), "Mismatch between TrieDictionarySearch and DoubleArrayTrieDictionarySearch"); + } + + + private List generateLargeDictionary() { + + return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString()) + .collect(Collectors.toList()); + } + + + static final class RandomStringGenerator { + + private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + private static final int STRING_LENGTH = 50; + private static final SecureRandom RANDOM = new SecureRandom(); + + + public static String generateRandomString() { + + StringBuilder sb = new StringBuilder(STRING_LENGTH); + for (int i = 0; i < STRING_LENGTH; i++) { + int index = RANDOM.nextInt(CHARACTERS.length()); + sb.append(CHARACTERS.charAt(index)); + } + return sb.toString(); + } + + } + + + @Test + public void testMultiplePayloads() { + + TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(Map.of(new DictionaryIdentifier("type1", EntityType.ENTITY, false, false), + List.of("apple", "banana"), + new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false), + List.of("apple", "orange"), + new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false), + List.of("apple", "kiwi"))); + + List dictionaryMatches = dictionarySearchImpl.getBoundariesAsList( + "an apple is delicious, a banana and a kiwi as well. orange is a color."); + + assertEquals(dictionaryMatches.size(), 6); + + } + + + @Test + public void testDoubleArrayTrie() { + + Map> map = new HashMap<>(); + String[] keyArray = new String[]{"hers", "his", "she", "he"}; + for (String key : keyArray) { + map.put(key, List.of(key, key, key)); + } + + AhoCorasickDoubleArrayTrie> acdat = new AhoCorasickDoubleArrayTrie<>(); + acdat.build(map); + + final String text = "uhers"; + List>> wordList = acdat.parseText(text); + assertEquals(wordList.size(), 2); + } + +}