RED-10290: Improve SearchImplementation logic for dictionaries
This commit is contained in:
maverickstuder
parent
4a3d1c9a0a
commit
f15db27a7c
@ -61,7 +61,8 @@ dependencies {
|
||||
|
||||
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
|
||||
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
|
||||
implementation("org.ahocorasick:ahocorasick:0.6.3")
|
||||
implementation("org.ahocorasick:ahocorasick:0.9.0")
|
||||
implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2")
|
||||
implementation("org.javassist:javassist:3.29.2-GA")
|
||||
|
||||
implementation("org.drools:drools-engine:${droolsVersion}")
|
||||
@ -87,7 +88,6 @@ dependencies {
|
||||
implementation("org.reflections:reflections:0.10.2")
|
||||
|
||||
implementation("com.opencsv:opencsv:5.9")
|
||||
implementation("com.gliwka.hyperscan:hyperscan:5.4.11-3.0.0")
|
||||
implementation("com.joestelmach:natty:0.13")
|
||||
|
||||
testImplementation(project(":rules-management"))
|
||||
|
||||
@ -26,7 +26,7 @@ public class RedactionServiceSettings {
|
||||
|
||||
private boolean llmNerServiceEnabled;
|
||||
|
||||
private boolean priorityMode;
|
||||
private boolean priorityMode = false;
|
||||
|
||||
private long firstLevelDictionaryCacheMaximumSize = 1000;
|
||||
|
||||
|
||||
@ -2,14 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@ -31,28 +24,70 @@ import lombok.Getter;
|
||||
@Data
|
||||
public class Dictionary {
|
||||
|
||||
private final Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
private final Map<String, Map<Level, DictionaryModel>> localAccessMap = new HashMap<>();
|
||||
|
||||
@Getter
|
||||
private final DictionaryVersion version;
|
||||
|
||||
private final DictionarySearch dictionarySearch;
|
||||
|
||||
public enum Level {
|
||||
DOSSIER_TEMPLATE,
|
||||
DOSSIER
|
||||
}
|
||||
|
||||
|
||||
Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version, DictionarySearch dictionarySearch) {
|
||||
|
||||
dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
|
||||
dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), Map.of(getLevel(dm.isDossierDictionary()), dm)));
|
||||
this.version = version;
|
||||
this.dictionarySearch = dictionarySearch;
|
||||
}
|
||||
|
||||
|
||||
public int getDictionaryRank(String type) {
|
||||
private Level getLevel(boolean isDossierDictionary) {
|
||||
|
||||
return isDossierDictionary ? Level.DOSSIER : Level.DOSSIER_TEMPLATE;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Determines the default level for a given type based on the levels present.
|
||||
* If both levels are present, it defaults to {@code Level.DOSSIER}.
|
||||
*
|
||||
* @param type The type to determine the default level for.
|
||||
* @return The default {@link Level} for the specified type.
|
||||
* @throws NotFoundException If the type is not found in the dictionary.
|
||||
*/
|
||||
private Level getDefaultLevel(String type) {
|
||||
|
||||
Map<Level, DictionaryModel> levelMap = localAccessMap.get(type);
|
||||
if (levelMap == null || levelMap.isEmpty()) {
|
||||
throw new NotFoundException("Type: " + type + " is not found");
|
||||
}
|
||||
if (levelMap.containsKey(Level.DOSSIER)) {
|
||||
return Level.DOSSIER;
|
||||
} else {
|
||||
// Use whatever level is present
|
||||
return levelMap.keySet()
|
||||
.iterator().next();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int getDictionaryRank(String type, Level level) {
|
||||
|
||||
if (!localAccessMap.containsKey(type)) {
|
||||
return 0;
|
||||
}
|
||||
return localAccessMap.get(type).getRank();
|
||||
DictionaryModel model = localAccessMap.get(type)
|
||||
.get(level);
|
||||
return model != null ? model.getRank() : 0;
|
||||
}
|
||||
|
||||
|
||||
public int getDictionaryRank(String type) {
|
||||
|
||||
return getDictionaryRank(type, getDefaultLevel(type));
|
||||
}
|
||||
|
||||
|
||||
@ -72,6 +107,8 @@ public class Dictionary {
|
||||
|
||||
return localAccessMap.values()
|
||||
.stream()
|
||||
.flatMap(levelDictionaryModelMap -> levelDictionaryModelMap.values()
|
||||
.stream())
|
||||
.toList();
|
||||
}
|
||||
|
||||
@ -83,56 +120,144 @@ public class Dictionary {
|
||||
|
||||
|
||||
/**
|
||||
* Retrieves the {@link DictionaryModel} of a specified type.
|
||||
* Retrieves the {@link DictionaryModel} of a specified type and level.
|
||||
*
|
||||
* @param type The type of dictionary model to retrieve.
|
||||
* @return The {@link DictionaryModel} of the specified type.
|
||||
* @throws NotFoundException If the specified type is not found in the dictionary.
|
||||
* @param type The type of dictionary model to retrieve.
|
||||
* @param level The level of the dictionary model to retrieve.
|
||||
* @return The {@link DictionaryModel} of the specified type and level.
|
||||
* @throws NotFoundException If the specified type or level is not found in the dictionary.
|
||||
*/
|
||||
public DictionaryModel getType(String type) {
|
||||
public DictionaryModel getType(String type, Level level) {
|
||||
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model == null) {
|
||||
throw new NotFoundException("Type: " + type + " is not found");
|
||||
Map<Level, DictionaryModel> levelMap = localAccessMap.get(type);
|
||||
if (levelMap == null || !levelMap.containsKey(level)) {
|
||||
throw new NotFoundException("Type: " + type + " with level: " + level + " is not found");
|
||||
}
|
||||
return model;
|
||||
return levelMap.get(level);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks if the dictionary of a specific type is considered a hint.
|
||||
* Retrieves the {@link DictionaryModel} of a specified type at the default level.
|
||||
*
|
||||
* @param type The type of dictionary model to retrieve.
|
||||
* @return The {@link DictionaryModel} of the specified type at the default level.
|
||||
* @throws NotFoundException If the specified type is not found in the dictionary.
|
||||
*/
|
||||
public DictionaryModel getType(String type) {
|
||||
|
||||
return getType(type, getDefaultLevel(type));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks if the dictionary of a specific type and level is considered a hint.
|
||||
*
|
||||
* @param type The type of dictionary to check.
|
||||
* @param level The level of the dictionary to check.
|
||||
* @return true if the dictionary model is marked as a hint, false otherwise.
|
||||
*/
|
||||
public boolean isHint(String type, Level level) {
|
||||
|
||||
DictionaryModel model = localAccessMap.get(type)
|
||||
.get(level);
|
||||
return model != null && model.isHint();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks if the dictionary of a specific type is considered a hint at the default level.
|
||||
*
|
||||
* @param type The type of dictionary to check.
|
||||
* @return true if the dictionary model is marked as a hint, false otherwise.
|
||||
*/
|
||||
public boolean isHint(String type) {
|
||||
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.isHint();
|
||||
}
|
||||
return false;
|
||||
return isHint(type, getDefaultLevel(type));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks if the dictionary of a specific type is case-insensitive.
|
||||
* Checks if the dictionary of a specific type and level is case-insensitive.
|
||||
*
|
||||
* @param type The type of dictionary to check.
|
||||
* @param level The level of the dictionary to check.
|
||||
* @return true if the dictionary is case-insensitive, false otherwise.
|
||||
*/
|
||||
public boolean isCaseInsensitiveDictionary(String type, Level level) {
|
||||
|
||||
DictionaryModel dictionaryModel = localAccessMap.get(type)
|
||||
.get(level);
|
||||
return dictionaryModel != null && dictionaryModel.isCaseInsensitive();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks if the dictionary of a specific type is case-insensitive at the default level.
|
||||
*
|
||||
* @param type The type of dictionary to check.
|
||||
* @return true if the dictionary is case-insensitive, false otherwise.
|
||||
*/
|
||||
public boolean isCaseInsensitiveDictionary(String type) {
|
||||
|
||||
DictionaryModel dictionaryModel = localAccessMap.get(type);
|
||||
if (dictionaryModel != null) {
|
||||
return dictionaryModel.isCaseInsensitive();
|
||||
}
|
||||
return false;
|
||||
return isCaseInsensitiveDictionary(type, getDefaultLevel(type));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Adds a local dictionary entry of a specific type.
|
||||
* Adds a local dictionary entry of a specific type and level.
|
||||
*
|
||||
* @param type The type of dictionary to add the entry to.
|
||||
* @param value The value of the entry.
|
||||
* @param matchedRules A collection of {@link MatchedRule} associated with the entry.
|
||||
* @param alsoAddLastname Indicates whether to also add the lastname separately as an entry.
|
||||
* @param level The level of the dictionary where the entry should be added.
|
||||
* @throws IllegalArgumentException If the specified type does not exist within the dictionary, if the type
|
||||
* does not have any local entries defined, or if the provided value is
|
||||
* blank. This ensures that only valid, non-empty entries
|
||||
* are added to the dictionary.
|
||||
*/
|
||||
private void addLocalDictionaryEntry(String type, String value, Collection<MatchedRule> matchedRules, boolean alsoAddLastname, Level level) {
|
||||
|
||||
if (value.isBlank()) {
|
||||
return;
|
||||
}
|
||||
Map<Level, DictionaryModel> levelMap = localAccessMap.get(type);
|
||||
if (levelMap == null || !levelMap.containsKey(level)) {
|
||||
throw new IllegalArgumentException(format("DictionaryModel of type %s with level %s does not exist", type, level));
|
||||
}
|
||||
DictionaryModel dictionaryModel = levelMap.get(level);
|
||||
if (dictionaryModel.getLocalEntriesWithMatchedRules() == null) {
|
||||
throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type));
|
||||
}
|
||||
if (StringUtils.isEmpty(value)) {
|
||||
throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
|
||||
}
|
||||
boolean isCaseInsensitive = dictionaryModel.isCaseInsensitive();
|
||||
Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
|
||||
|
||||
String cleanedValue = value;
|
||||
if (isCaseInsensitive) {
|
||||
cleanedValue = cleanedValue.toLowerCase(Locale.US);
|
||||
}
|
||||
dictionaryModel.getLocalEntriesWithMatchedRules()
|
||||
.merge(cleanedValue.trim(),
|
||||
matchedRulesSet,
|
||||
(set1, set2) -> Stream.concat(set1.stream(), set2.stream())
|
||||
.collect(Collectors.toSet()));
|
||||
if (alsoAddLastname) {
|
||||
String lastname = cleanedValue.split(" ")[0];
|
||||
dictionaryModel.getLocalEntriesWithMatchedRules()
|
||||
.merge(lastname,
|
||||
matchedRulesSet,
|
||||
(set1, set2) -> Stream.concat(set1.stream(), set2.stream())
|
||||
.collect(Collectors.toSet()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Adds a local dictionary entry of a specific type at the default level.
|
||||
*
|
||||
* @param type The type of dictionary to add the entry to.
|
||||
* @param value The value of the entry.
|
||||
@ -145,40 +270,7 @@ public class Dictionary {
|
||||
*/
|
||||
private void addLocalDictionaryEntry(String type, String value, Collection<MatchedRule> matchedRules, boolean alsoAddLastname) {
|
||||
|
||||
if (value.isBlank()) {
|
||||
return;
|
||||
}
|
||||
if (localAccessMap.get(type) == null) {
|
||||
throw new IllegalArgumentException(format("DictionaryModel of type %s does not exist", type));
|
||||
}
|
||||
if (localAccessMap.get(type).getLocalEntriesWithMatchedRules() == null) {
|
||||
throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type));
|
||||
}
|
||||
if (StringUtils.isEmpty(value)) {
|
||||
throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
|
||||
}
|
||||
boolean isCaseInsensitive = localAccessMap.get(type).isCaseInsensitive();
|
||||
Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
|
||||
|
||||
String cleanedValue = value;
|
||||
if (isCaseInsensitive) {
|
||||
cleanedValue = cleanedValue.toLowerCase(Locale.US);
|
||||
}
|
||||
localAccessMap.get(type)
|
||||
.getLocalEntriesWithMatchedRules()
|
||||
.merge(cleanedValue.trim(),
|
||||
matchedRulesSet,
|
||||
(set1, set2) -> Stream.concat(set1.stream(), set2.stream())
|
||||
.collect(Collectors.toSet()));
|
||||
if (alsoAddLastname) {
|
||||
String lastname = cleanedValue.split(" ")[0];
|
||||
localAccessMap.get(type)
|
||||
.getLocalEntriesWithMatchedRules()
|
||||
.merge(lastname,
|
||||
matchedRulesSet,
|
||||
(set1, set2) -> Stream.concat(set1.stream(), set2.stream())
|
||||
.collect(Collectors.toSet()));
|
||||
}
|
||||
addLocalDictionaryEntry(type, value, matchedRules, alsoAddLastname, getDefaultLevel(type));
|
||||
}
|
||||
|
||||
|
||||
@ -186,10 +278,22 @@ public class Dictionary {
|
||||
* Recommends a text entity for inclusion in every dictionary model without separating the last name.
|
||||
*
|
||||
* @param textEntity The {@link TextEntity} to be recommended.
|
||||
* @param level The level of the dictionary where the recommendation should be added.
|
||||
*/
|
||||
public void recommendEverywhere(TextEntity textEntity, Level level) {
|
||||
|
||||
addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false, level);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Recommends a text entity for inclusion in every dictionary model without separating the last name at the default level.
|
||||
*
|
||||
* @param textEntity The {@link TextEntity} to be recommended.
|
||||
*/
|
||||
public void recommendEverywhere(TextEntity textEntity) {
|
||||
|
||||
addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
|
||||
recommendEverywhere(textEntity, getDefaultLevel(textEntity.type()));
|
||||
}
|
||||
|
||||
|
||||
@ -197,10 +301,22 @@ public class Dictionary {
|
||||
* Recommends a text entity for inclusion in every dictionary model with the last name added separately.
|
||||
*
|
||||
* @param textEntity The {@link TextEntity} to be recommended.
|
||||
* @param level The level of the dictionary where the recommendation should be added.
|
||||
*/
|
||||
public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity, Level level) {
|
||||
|
||||
addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true, level);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Recommends a text entity for inclusion in every dictionary model with the last name added separately at the default level.
|
||||
*
|
||||
* @param textEntity The {@link TextEntity} to be recommended.
|
||||
*/
|
||||
public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) {
|
||||
|
||||
addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
|
||||
recommendEverywhereWithLastNameSeparately(textEntity, getDefaultLevel(textEntity.type()));
|
||||
}
|
||||
|
||||
|
||||
@ -208,11 +324,22 @@ public class Dictionary {
|
||||
* Adds multiple author names contained within a text entity as recommendations in the dictionary.
|
||||
*
|
||||
* @param textEntity The {@link TextEntity} containing author names to be added.
|
||||
* @param level The level of the dictionary where the recommendations should be added.
|
||||
*/
|
||||
public void addMultipleAuthorsAsRecommendation(TextEntity textEntity, Level level) {
|
||||
|
||||
splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true, level));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Adds multiple author names contained within a text entity as recommendations in the dictionary at the default level.
|
||||
*
|
||||
* @param textEntity The {@link TextEntity} containing author names to be added.
|
||||
*/
|
||||
public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
|
||||
|
||||
splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true));
|
||||
|
||||
addMultipleAuthorsAsRecommendation(textEntity, getDefaultLevel(textEntity.type()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,13 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -17,24 +19,12 @@ import lombok.SneakyThrows;
|
||||
@RequiredArgsConstructor
|
||||
public class DictionaryFactory {
|
||||
|
||||
private final RedactionServiceSettings settings;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public Dictionary create(List<DictionaryModel> dictionaryModels, DictionaryVersion dictionaryVersion) {
|
||||
|
||||
// Generate the combined dictionary values map
|
||||
Map<DictionaryIdentifier, List<String>> combinedMap = generateCombinedDictionaryValuesMap(dictionaryModels);
|
||||
|
||||
// Determine which DictionarySearch implementation to use
|
||||
DictionarySearch dictionarySearch;
|
||||
if (settings.isPriorityMode()) {
|
||||
dictionarySearch = new HyperscanDictionarySearch(combinedMap);
|
||||
} else {
|
||||
dictionarySearch = new TrieDictionarySearch(combinedMap);
|
||||
}
|
||||
|
||||
// Instantiate and return the Dictionary
|
||||
DictionarySearch dictionarySearch = new DoubleArrayTrieDictionarySearch(combinedMap);
|
||||
return new Dictionary(dictionaryModels, dictionaryVersion, dictionarySearch);
|
||||
}
|
||||
|
||||
@ -42,62 +32,45 @@ public class DictionaryFactory {
|
||||
private Map<DictionaryIdentifier, List<String>> generateCombinedDictionaryValuesMap(List<DictionaryModel> dictionaryModels) {
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> combinedValuesMap = new HashMap<>();
|
||||
|
||||
for (DictionaryModel model : dictionaryModels) {
|
||||
|
||||
DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive());
|
||||
DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(),
|
||||
EntityType.DICTIONARY_REMOVAL,
|
||||
model.isDossierDictionary(),
|
||||
!model.isCaseInsensitive());
|
||||
DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(),
|
||||
EntityType.FALSE_POSITIVE,
|
||||
model.isDossierDictionary(),
|
||||
!model.isCaseInsensitive());
|
||||
DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(),
|
||||
EntityType.FALSE_RECOMMENDATION,
|
||||
model.isDossierDictionary(),
|
||||
!model.isCaseInsensitive());
|
||||
|
||||
List<String> entryValues = model.getEntries()
|
||||
.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
|
||||
List<String> deletedEntryValues = model.getEntries()
|
||||
.stream()
|
||||
.filter(DictionaryEntry::isDeleted)
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
|
||||
List<String> falsePositiveValues = model.getFalsePositives()
|
||||
.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
|
||||
List<String> falseRecommendationValues = model.getFalseRecommendations()
|
||||
.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
|
||||
if (!entryValues.isEmpty()) {
|
||||
combinedValuesMap.put(entriesIdentifier, entryValues);
|
||||
}
|
||||
if (!deletedEntryValues.isEmpty()) {
|
||||
combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues);
|
||||
}
|
||||
if (!falsePositiveValues.isEmpty()) {
|
||||
combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues);
|
||||
}
|
||||
if (!falseRecommendationValues.isEmpty()) {
|
||||
combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues);
|
||||
}
|
||||
addDictionaryEntries(combinedValuesMap, model);
|
||||
}
|
||||
|
||||
return combinedValuesMap;
|
||||
}
|
||||
|
||||
|
||||
private void addDictionaryEntries(Map<DictionaryIdentifier, List<String>> combinedValuesMap, DictionaryModel model) {
|
||||
|
||||
addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.ENTITY), filterValues(model.getEntries(), false));
|
||||
addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.FALSE_POSITIVE), filterValues(model.getFalsePositives(), false));
|
||||
addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.FALSE_RECOMMENDATION), filterValues(model.getFalseRecommendations(), false));
|
||||
|
||||
if (model.isDossierDictionary()) {
|
||||
addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.DICTIONARY_REMOVAL), filterValues(model.getEntries(), true));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private DictionaryIdentifier createIdentifier(DictionaryModel model, EntityType entityType) {
|
||||
|
||||
return new DictionaryIdentifier(model.getType(), entityType, model.isDossierDictionary(), !model.isCaseInsensitive());
|
||||
}
|
||||
|
||||
|
||||
private List<String> filterValues(Set<DictionaryEntryModel> entries, boolean isDeleted) {
|
||||
|
||||
return entries.stream()
|
||||
.filter(entry -> entry.isDeleted() == isDeleted)
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
private void addValuesToMap(Map<DictionaryIdentifier, List<String>> map, DictionaryIdentifier key, List<String> values) {
|
||||
|
||||
if (!values.isEmpty()) {
|
||||
map.computeIfAbsent(key, k -> new ArrayList<>()).addAll(values);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -5,10 +5,10 @@ import org.ahocorasick.trie.PayloadTrie;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
public class DictionaryTrie {
|
||||
public class DictionaryIdentifierTrie {
|
||||
private final PayloadTrie<DictionaryIdentifier> trie;
|
||||
|
||||
private DictionaryTrie(PayloadTrie<DictionaryIdentifier> trie) {
|
||||
private DictionaryIdentifierTrie(PayloadTrie<DictionaryIdentifier> trie) {
|
||||
this.trie = trie;
|
||||
}
|
||||
|
||||
@ -36,8 +36,8 @@ public class DictionaryTrie {
|
||||
return this;
|
||||
}
|
||||
|
||||
public DictionaryTrie build() {
|
||||
return new DictionaryTrie(builder.build());
|
||||
public DictionaryIdentifierTrie build() {
|
||||
return new DictionaryIdentifierTrie(builder.build());
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,14 +10,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
|
||||
*/
|
||||
public interface DictionarySearch {
|
||||
|
||||
/**
|
||||
* Checks if at least one dictionary entry matches the given text.
|
||||
*
|
||||
* @param text The text to search within.
|
||||
* @return True if at least one match is found, otherwise false.
|
||||
*/
|
||||
boolean atLeastOneMatches(String text);
|
||||
|
||||
/**
|
||||
* Retrieves a list of match boundaries within the given text.
|
||||
*
|
||||
|
||||
@ -0,0 +1,126 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
|
||||
public class DoubleArrayTrieDictionarySearch implements DictionarySearch {
|
||||
|
||||
private final AhoCorasickDoubleArrayTrie<List<DictionaryIdentifierWithKeyword>> trie;
|
||||
|
||||
|
||||
public DoubleArrayTrieDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
|
||||
trie = new AhoCorasickDoubleArrayTrie<>();
|
||||
trie.build(computeStringIdentifiersMap(dictionaryValues));
|
||||
}
|
||||
|
||||
|
||||
private static Map<String, List<DictionaryIdentifierWithKeyword>> computeStringIdentifiersMap(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
|
||||
Map<String, List<DictionaryIdentifierWithKeyword>> stringToIdentifiersMap = new HashMap<>();
|
||||
for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
|
||||
DictionaryIdentifier identifier = entry.getKey();
|
||||
List<String> values = entry.getValue();
|
||||
for (String value : values) {
|
||||
DictionaryIdentifierWithKeyword idWithKeyword = new DictionaryIdentifierWithKeyword(identifier, value);
|
||||
stringToIdentifiersMap.computeIfAbsent(value.toLowerCase(Locale.ROOT), k -> new ArrayList<>()).add(idWithKeyword);
|
||||
}
|
||||
}
|
||||
return stringToIdentifiersMap;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text) {
|
||||
|
||||
String lowerText = text.toString().toLowerCase(Locale.ROOT);
|
||||
return getMatchTextRangeStream(text, lowerText, 0);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
|
||||
|
||||
return getBoundaries(textBlock, textBlock.getTextRange());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
|
||||
|
||||
String subText = text.subSequence(region.start(), region.end()).toString();
|
||||
String lowerSubText = subText.toLowerCase(Locale.ROOT);
|
||||
return getMatchTextRangeStream(text, lowerSubText, region.start());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchPosition> getMatches(String text) {
|
||||
|
||||
List<MatchPosition> matches = new ArrayList<>();
|
||||
String lowerText = text.toLowerCase(Locale.ROOT);
|
||||
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(lowerText);
|
||||
|
||||
for (AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit : hits) {
|
||||
String matchedText = text.substring(hit.begin, hit.end);
|
||||
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
|
||||
|
||||
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
|
||||
MatchPosition matchPosition = new MatchPosition(idkw.identifier, hit.begin, hit.end);
|
||||
if (idkw.identifier.caseSensitive()) {
|
||||
if (matchedText.equals(idkw.keyword)) {
|
||||
matches.add(matchPosition);
|
||||
}
|
||||
} else {
|
||||
matches.add(matchPosition);
|
||||
}
|
||||
}
|
||||
}
|
||||
return matches.stream();
|
||||
}
|
||||
|
||||
|
||||
private Stream<MatchTextRange> getMatchTextRangeStream(CharSequence text, String lowerText, int offset) {
|
||||
|
||||
List<MatchTextRange> matches = new ArrayList<>();
|
||||
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(lowerText);
|
||||
|
||||
for (AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit : hits) {
|
||||
addMatchesForHit(text, matches, hit, offset);
|
||||
}
|
||||
return matches.stream();
|
||||
}
|
||||
|
||||
|
||||
private void addMatchesForHit(CharSequence text, List<MatchTextRange> matches, AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit, int offset) {
|
||||
|
||||
int start = hit.begin + offset;
|
||||
int end = hit.end + offset;
|
||||
String matchedText = text.subSequence(start, end).toString();
|
||||
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
|
||||
|
||||
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
|
||||
if (idkw.identifier.caseSensitive()) {
|
||||
if (matchedText.equals(idkw.keyword)) {
|
||||
matches.add(new MatchTextRange(idkw.identifier, new TextRange(start, end)));
|
||||
}
|
||||
} else {
|
||||
matches.add(new MatchTextRange(idkw.identifier, new TextRange(start, end)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private record DictionaryIdentifierWithKeyword(DictionaryIdentifier identifier, String keyword) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,172 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.gliwka.hyperscan.wrapper.CompileErrorException;
|
||||
import com.gliwka.hyperscan.wrapper.Database;
|
||||
import com.gliwka.hyperscan.wrapper.Expression;
|
||||
import com.gliwka.hyperscan.wrapper.ExpressionFlag;
|
||||
import com.gliwka.hyperscan.wrapper.Match;
|
||||
import com.gliwka.hyperscan.wrapper.Scanner;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
|
||||
public class HyperscanDictionarySearch implements DictionarySearch {
|
||||
|
||||
private final Map<Integer, DictionaryIdentifier> patternIdMap = new ConcurrentHashMap<>();
|
||||
private AtomicInteger patternIdCounter = new AtomicInteger(0);
|
||||
private Database database;
|
||||
|
||||
|
||||
public HyperscanDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) throws CompileErrorException {
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> entries = new HashMap<>(dictionaryValues);
|
||||
this.database = compileDatabase(entries, this.patternIdMap);
|
||||
}
|
||||
|
||||
|
||||
private Database compileDatabase(Map<DictionaryIdentifier, List<String>> entries, Map<Integer, DictionaryIdentifier> patternIdMap) throws CompileErrorException {
|
||||
|
||||
if (entries.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
LinkedList<Expression> expressions = new LinkedList<>();
|
||||
|
||||
for (Map.Entry<DictionaryIdentifier, List<String>> entry : entries.entrySet()) {
|
||||
DictionaryIdentifier identifier = entry.getKey();
|
||||
for (String value : entry.getValue()) {
|
||||
int patternId = patternIdCounter.incrementAndGet();
|
||||
patternIdMap.put(patternId, identifier);
|
||||
|
||||
EnumSet<ExpressionFlag> flags = EnumSet.noneOf(ExpressionFlag.class);
|
||||
if (!identifier.caseSensitive()) {
|
||||
flags.add(ExpressionFlag.CASELESS);
|
||||
}
|
||||
// escape regex characters as we only use literals
|
||||
expressions.add(new Expression(Pattern.quote(value), flags, patternId));
|
||||
}
|
||||
}
|
||||
|
||||
// Build the database
|
||||
return Database.compile(expressions);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean atLeastOneMatches(String text) {
|
||||
|
||||
if (this.database == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try (Scanner scanner = new Scanner()) {
|
||||
scanner.allocScratch(this.database);
|
||||
List<Match> matches = scanner.scan(this.database, text);
|
||||
return !matches.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text) {
|
||||
|
||||
List<MatchTextRange> boundaries = new ArrayList<>();
|
||||
|
||||
if (this.database == null) {
|
||||
return Stream.empty();
|
||||
}
|
||||
|
||||
try (Scanner scanner = new Scanner()) {
|
||||
scanner.allocScratch(this.database);
|
||||
List<Match> matches = scanner.scan(this.database, text.toString());
|
||||
for (Match match : matches) {
|
||||
int start = (int) match.getStartPosition();
|
||||
int end = (int) match.getEndPosition();
|
||||
int patternId = match.getMatchedExpression().getId();
|
||||
DictionaryIdentifier identifier = patternIdMap.get(patternId);
|
||||
boundaries.add(new MatchTextRange(identifier, new TextRange(start, end)));
|
||||
}
|
||||
}
|
||||
|
||||
return boundaries.stream();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
|
||||
|
||||
return getBoundaries(textBlock, textBlock.getTextRange());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
|
||||
|
||||
List<MatchTextRange> boundaries = new ArrayList<>();
|
||||
|
||||
if (this.database == null) {
|
||||
return Stream.empty();
|
||||
}
|
||||
|
||||
CharSequence subText = text.subSequence(region.start(), region.end());
|
||||
|
||||
try (Scanner scanner = new Scanner()) {
|
||||
scanner.allocScratch(this.database);
|
||||
List<Match> matches = scanner.scan(this.database, subText.toString());
|
||||
for (Match match : matches) {
|
||||
int start = (int) (match.getStartPosition() + region.start());
|
||||
int end = (int) (match.getEndPosition() + region.start());
|
||||
int patternId = match.getMatchedExpression().getId();
|
||||
DictionaryIdentifier identifier = patternIdMap.get(patternId);
|
||||
boundaries.add(new MatchTextRange(identifier, new TextRange(start, end)));
|
||||
}
|
||||
}
|
||||
|
||||
return boundaries.stream();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchPosition> getMatches(String text) {
|
||||
|
||||
List<MatchPosition> matchPositions = new ArrayList<>();
|
||||
|
||||
if (this.database == null) {
|
||||
return Stream.empty();
|
||||
}
|
||||
|
||||
try (Scanner scanner = new Scanner()) {
|
||||
scanner.allocScratch(this.database);
|
||||
List<Match> matches = scanner.scan(this.database, text);
|
||||
for (Match match : matches) {
|
||||
int start = (int) match.getStartPosition();
|
||||
int end = (int) match.getEndPosition();
|
||||
int patternId = match.getMatchedExpression().getId();
|
||||
DictionaryIdentifier identifier = patternIdMap.get(patternId);
|
||||
matchPositions.add(new MatchPosition(identifier, start, end));
|
||||
}
|
||||
}
|
||||
|
||||
return matchPositions.stream();
|
||||
}
|
||||
|
||||
|
||||
public void close() {
|
||||
|
||||
if (this.database != null) {
|
||||
this.database.close();
|
||||
this.database = null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -4,7 +4,6 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
@ -12,15 +11,14 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
|
||||
|
||||
public class TrieDictionarySearch implements DictionarySearch {
|
||||
|
||||
private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new ConcurrentHashMap<>();
|
||||
private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new ConcurrentHashMap<>();
|
||||
private final DictionaryTrie caseSensitiveTrie;
|
||||
private final DictionaryTrie caseInsensitiveTrie;
|
||||
private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new HashMap<>();
|
||||
private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new HashMap<>();
|
||||
private final DictionaryIdentifierTrie caseSensitiveTrie;
|
||||
private final DictionaryIdentifierTrie caseInsensitiveTrie;
|
||||
|
||||
|
||||
public TrieDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
|
||||
System.out.println("TEST ---> DictionarySearchImplementation CTOR");
|
||||
for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
|
||||
DictionaryIdentifier identifier = entry.getKey();
|
||||
List<String> values = entry.getValue();
|
||||
@ -36,12 +34,12 @@ public class TrieDictionarySearch implements DictionarySearch {
|
||||
}
|
||||
|
||||
|
||||
private DictionaryTrie createTrie(Map<DictionaryIdentifier, List<String>> entries, boolean ignoreCase) {
|
||||
private DictionaryIdentifierTrie createTrie(Map<DictionaryIdentifier, List<String>> entries, boolean ignoreCase) {
|
||||
|
||||
if (entries.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
DictionaryTrie.Builder builder = new DictionaryTrie.Builder();
|
||||
DictionaryIdentifierTrie.Builder builder = new DictionaryIdentifierTrie.Builder();
|
||||
if (ignoreCase) {
|
||||
builder.ignoreCase();
|
||||
}
|
||||
@ -54,7 +52,6 @@ public class TrieDictionarySearch implements DictionarySearch {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean atLeastOneMatches(String text) {
|
||||
|
||||
if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) {
|
||||
@ -101,7 +98,7 @@ public class TrieDictionarySearch implements DictionarySearch {
|
||||
}
|
||||
|
||||
|
||||
private void addMatchTextRangesForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchTextRange> matches, CharSequence text) {
|
||||
private void addMatchTextRangesForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryIdentifierTrie trie, List<MatchTextRange> matches, CharSequence text) {
|
||||
|
||||
if (!entries.isEmpty() && trie != null) {
|
||||
matches.addAll(trie.parseText(text)
|
||||
@ -116,7 +113,7 @@ public class TrieDictionarySearch implements DictionarySearch {
|
||||
TextRange region,
|
||||
List<MatchTextRange> matches,
|
||||
Map<DictionaryIdentifier, List<String>> entries,
|
||||
DictionaryTrie trie) {
|
||||
DictionaryIdentifierTrie trie) {
|
||||
|
||||
if (!entries.isEmpty() && trie != null) {
|
||||
CharSequence subSequence = text.subSequence(region.start(), region.end());
|
||||
@ -128,7 +125,7 @@ public class TrieDictionarySearch implements DictionarySearch {
|
||||
}
|
||||
|
||||
|
||||
private void addMatchPositionsForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchPosition> matches, String text) {
|
||||
private void addMatchPositionsForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryIdentifierTrie trie, List<MatchPosition> matches, String text) {
|
||||
|
||||
if (!entries.isEmpty() && trie != null) {
|
||||
matches.addAll(trie.parseText(text)
|
||||
|
||||
@ -31,7 +31,7 @@ public class DictionarySearchService {
|
||||
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
|
||||
public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
|
||||
|
||||
semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node));
|
||||
semanticNodes.parallelStream().forEach(node -> addDictionaryEntities(dictionary, node));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -223,7 +223,7 @@ public class DictionaryService {
|
||||
combinedEntries,
|
||||
combinedFalsePositives,
|
||||
combinedFalseRecommendations,
|
||||
type.isDossierDictionaryOnly());
|
||||
dossierId != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -58,21 +58,6 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
|
||||
@Autowired
|
||||
DictionaryCacheService dictionaryCacheService;
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||
@Import(LayoutParsingServiceProcessorConfiguration.class)
|
||||
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
|
||||
public static class RedactionIntegrationTestConfiguration {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public StorageService inmemoryStorage() {
|
||||
|
||||
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@BeforeEach
|
||||
public void stubClients() {
|
||||
|
||||
@ -1,147 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
|
||||
import static org.springframework.test.util.AssertionErrors.assertEquals;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.HyperscanDictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.gliwka.hyperscan.wrapper.CompileErrorException;
|
||||
|
||||
@Disabled
|
||||
public class DictionaryPerformanceTest {
|
||||
|
||||
private static final int LARGE_DICTIONARY_SIZE = 5_000;
|
||||
private static final int LARGE_TEXT_REPETITIONS = 50_000;
|
||||
private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
|
||||
+ "Entity_1 match text. Recommendation_1 also here. Random text continues. ";
|
||||
|
||||
@Test
|
||||
public void testLargeFileWithLargeDictionaries() {
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
|
||||
|
||||
// Generate dictionary values with specific terms for matching
|
||||
IntStream.range(0, 6)
|
||||
.forEach(i -> {
|
||||
EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION;
|
||||
boolean caseSensitive = i % 2 == 0;
|
||||
|
||||
DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true,
|
||||
caseSensitive);
|
||||
List<String> dictionary = generateLargeDictionary();
|
||||
|
||||
// Add specific terms that are included in the large text for matches
|
||||
if (i == 0) {
|
||||
dictionary.add("Entity_1");
|
||||
}
|
||||
if (i == 1) {
|
||||
dictionary.add("recommendation_1");
|
||||
}
|
||||
dictionaryValues.put(identifier, dictionary);
|
||||
});
|
||||
|
||||
// Measure construction time for DictionarySearchImplementation
|
||||
long dictionaryTrieConstructionStart = System.currentTimeMillis();
|
||||
TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(dictionaryValues);
|
||||
long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart;
|
||||
|
||||
// Measure construction time for SearchImplementations
|
||||
long searchTrieConstructionStart = System.currentTimeMillis();
|
||||
List<SearchImplementation> searchImplementations = dictionaryValues.entrySet().stream()
|
||||
.map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive())).toList();
|
||||
long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart;
|
||||
|
||||
// Measure construction time for HyperSearchImplementation
|
||||
long hyperSearchConstructionStart = System.currentTimeMillis();
|
||||
HyperscanDictionarySearch hyperSearchImpl = null;
|
||||
try {
|
||||
hyperSearchImpl = new HyperscanDictionarySearch(dictionaryValues);
|
||||
} catch (CompileErrorException e) {
|
||||
assert false : "Failed to compile HyperSearchImplementation: " + e.getMessage();
|
||||
}
|
||||
long hyperSearchConstructionDuration = System.currentTimeMillis() - hyperSearchConstructionStart;
|
||||
|
||||
String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
|
||||
|
||||
// Measure search time for DictionarySearchImplementation
|
||||
long dictionarySearchStart = System.currentTimeMillis();
|
||||
List<TrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl
|
||||
.getBoundariesAsList(largeText);
|
||||
long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart;
|
||||
|
||||
// Measure search time for SearchImplementations
|
||||
long searchImplStart = System.currentTimeMillis();
|
||||
List<TextRange> searchMatches = new ArrayList<>();
|
||||
for (SearchImplementation searchImpl : searchImplementations) {
|
||||
searchMatches.addAll(searchImpl.getBoundaries(largeText));
|
||||
}
|
||||
long searchImplDuration = System.currentTimeMillis() - searchImplStart;
|
||||
|
||||
// Measure search time for HyperSearchImplementation
|
||||
long hyperSearchStart = System.currentTimeMillis();
|
||||
List<DictionarySearch.MatchPosition> hyperMatches = hyperSearchImpl.getMatchesAsList(largeText);
|
||||
long hyperSearchDuration = System.currentTimeMillis() - hyperSearchStart;
|
||||
|
||||
System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration);
|
||||
System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration,
|
||||
dictionaryMatches.size());
|
||||
System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration);
|
||||
System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration,
|
||||
searchMatches.size());
|
||||
System.out.printf("HyperSearchImplementation construction took %d ms\n", hyperSearchConstructionDuration);
|
||||
System.out.printf("HyperSearchImplementation search took %d ms and found %d matches\n", hyperSearchDuration,
|
||||
hyperMatches.size());
|
||||
|
||||
assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() && !hyperMatches.isEmpty()
|
||||
: "All implementations should find entities.";
|
||||
|
||||
// Ensure all implementations found the same number of matches
|
||||
assertEquals("DictionarySearchImplementation and SearchImplementations should find the same number of matches",
|
||||
dictionaryMatches.size(), searchMatches.size());
|
||||
assertEquals("DictionarySearchImplementation and HyperSearchImplementation should find the same number of matches",
|
||||
dictionaryMatches.size(), hyperMatches.size());
|
||||
|
||||
// Close the HyperSearchImplementation to release resources
|
||||
hyperSearchImpl.close();
|
||||
}
|
||||
|
||||
private List<String> generateLargeDictionary() {
|
||||
return IntStream.range(0, LARGE_DICTIONARY_SIZE)
|
||||
.mapToObj(i -> RandomStringGenerator.generateRandomString())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
static final class RandomStringGenerator {
|
||||
|
||||
private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
||||
private static final int STRING_LENGTH = 50;
|
||||
private static final SecureRandom RANDOM = new SecureRandom();
|
||||
|
||||
public static String generateRandomString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder(STRING_LENGTH);
|
||||
for (int i = 0; i < STRING_LENGTH; i++) {
|
||||
int index = RANDOM.nextInt(CHARACTERS.length());
|
||||
sb.append(CHARACTERS.charAt(index));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,176 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DoubleArrayTrieDictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
|
||||
public class DictionarySearchImplementationsTest {
|
||||
|
||||
private static final int LARGE_DICTIONARY_SIZE = 4_000;
|
||||
private static final int LARGE_TEXT_REPETITIONS = 500_000;
|
||||
private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
|
||||
+ "Entity_1 match text. Recommendation_1 also here. Random text continues. ";
|
||||
|
||||
|
||||
@Test
|
||||
//@Disabled
|
||||
public void performanceTest() {
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
|
||||
|
||||
// Generate dictionary values with specific terms for matching
|
||||
IntStream.range(0, 6)
|
||||
.forEach(i -> {
|
||||
EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION;
|
||||
boolean caseSensitive = i % 2 == 0;
|
||||
|
||||
DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive);
|
||||
List<String> dictionary = generateLargeDictionary();
|
||||
|
||||
// Add specific terms that are included in the large text for matches
|
||||
if (i == 0) {
|
||||
dictionary.add("Entity_1");
|
||||
}
|
||||
if (i == 1) {
|
||||
dictionary.add("recommendation_1");
|
||||
}
|
||||
dictionaryValues.put(identifier, dictionary);
|
||||
});
|
||||
|
||||
// Measure construction time for TrieDictionarySearch
|
||||
long trieDictionaryConstructionStart = System.currentTimeMillis();
|
||||
TrieDictionarySearch trieDictionarySearchImpl = new TrieDictionarySearch(dictionaryValues);
|
||||
long trieDictionaryConstructionDuration = System.currentTimeMillis() - trieDictionaryConstructionStart;
|
||||
|
||||
// Measure construction time for SearchImplementations
|
||||
long searchTrieConstructionStart = System.currentTimeMillis();
|
||||
List<SearchImplementation> searchImplementations = dictionaryValues.entrySet()
|
||||
.stream()
|
||||
.map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive()))
|
||||
.toList();
|
||||
long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart;
|
||||
|
||||
// Measure construction time for DoubleArrayTrieDictionarySearch
|
||||
long doubleArrayTrieConstructionStart = System.currentTimeMillis();
|
||||
DoubleArrayTrieDictionarySearch doubleArrayTrieSearchImpl = new DoubleArrayTrieDictionarySearch(dictionaryValues);
|
||||
long doubleArrayTrieConstructionDuration = System.currentTimeMillis() - doubleArrayTrieConstructionStart;
|
||||
|
||||
String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
|
||||
|
||||
// Measure search time for TrieDictionarySearch
|
||||
long trieDictionarySearchStart = System.currentTimeMillis();
|
||||
List<TrieDictionarySearch.MatchTextRange> trieDictionaryMatches = trieDictionarySearchImpl.getBoundariesAsList(largeText);
|
||||
long trieDictionarySearchDuration = System.currentTimeMillis() - trieDictionarySearchStart;
|
||||
|
||||
// Measure search time for SearchImplementations
|
||||
long searchImplStart = System.currentTimeMillis();
|
||||
List<TextRange> searchMatches = new ArrayList<>();
|
||||
for (SearchImplementation searchImpl : searchImplementations) {
|
||||
searchMatches.addAll(searchImpl.getBoundaries(largeText));
|
||||
}
|
||||
long searchImplDuration = System.currentTimeMillis() - searchImplStart;
|
||||
|
||||
// Measure search time for DoubleArrayTrieDictionarySearch
|
||||
long doubleArrayTrieSearchStart = System.currentTimeMillis();
|
||||
List<DictionarySearch.MatchTextRange> doubleArrayTrieMatches = doubleArrayTrieSearchImpl.getBoundariesAsList(largeText);
|
||||
long doubleArrayTrieSearchDuration = System.currentTimeMillis() - doubleArrayTrieSearchStart;
|
||||
|
||||
// Output the performance results
|
||||
System.out.printf("TrieDictionarySearch construction took %d ms\n", trieDictionaryConstructionDuration);
|
||||
System.out.printf("TrieDictionarySearch search took %d ms and found %d matches\n", trieDictionarySearchDuration, trieDictionaryMatches.size());
|
||||
System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration);
|
||||
System.out.printf("Combined SearchImplementation search took %d ms and found %d matches\n", searchImplDuration, searchMatches.size());
|
||||
System.out.printf("DoubleArrayTrieDictionarySearch construction took %d ms\n", doubleArrayTrieConstructionDuration);
|
||||
System.out.printf("DoubleArrayTrieDictionarySearch search took %d ms and found %d matches\n", doubleArrayTrieSearchDuration, doubleArrayTrieMatches.size());
|
||||
|
||||
// Assert that all implementations found matches
|
||||
assert !trieDictionaryMatches.isEmpty()
|
||||
&& !searchMatches.isEmpty()
|
||||
&& !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities.";
|
||||
|
||||
// Ensure all implementations found the same number of matches
|
||||
assertEquals(trieDictionaryMatches.size(), searchMatches.size(), "Mismatch between TrieDictionarySearch and SearchImplementations");
|
||||
assertEquals(trieDictionaryMatches.size(), doubleArrayTrieMatches.size(), "Mismatch between TrieDictionarySearch and DoubleArrayTrieDictionarySearch");
|
||||
}
|
||||
|
||||
|
||||
private List<String> generateLargeDictionary() {
|
||||
|
||||
return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
static final class RandomStringGenerator {
|
||||
|
||||
private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
||||
private static final int STRING_LENGTH = 50;
|
||||
private static final SecureRandom RANDOM = new SecureRandom();
|
||||
|
||||
|
||||
public static String generateRandomString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder(STRING_LENGTH);
|
||||
for (int i = 0; i < STRING_LENGTH; i++) {
|
||||
int index = RANDOM.nextInt(CHARACTERS.length());
|
||||
sb.append(CHARACTERS.charAt(index));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMultiplePayloads() {
|
||||
|
||||
TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(Map.of(new DictionaryIdentifier("type1", EntityType.ENTITY, false, false),
|
||||
List.of("apple", "banana"),
|
||||
new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false),
|
||||
List.of("apple", "orange"),
|
||||
new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false),
|
||||
List.of("apple", "kiwi")));
|
||||
|
||||
List<TrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundariesAsList(
|
||||
"an apple is delicious, a banana and a kiwi as well. orange is a color.");
|
||||
|
||||
assertEquals(dictionaryMatches.size(), 6);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testDoubleArrayTrie() {
|
||||
|
||||
Map<String, List<String>> map = new HashMap<>();
|
||||
String[] keyArray = new String[]{"hers", "his", "she", "he"};
|
||||
for (String key : keyArray) {
|
||||
map.put(key, List.of(key, key, key));
|
||||
}
|
||||
|
||||
AhoCorasickDoubleArrayTrie<List<String>> acdat = new AhoCorasickDoubleArrayTrie<>();
|
||||
acdat.build(map);
|
||||
|
||||
final String text = "uhers";
|
||||
List<AhoCorasickDoubleArrayTrie.Hit<List<String>>> wordList = acdat.parseText(text);
|
||||
assertEquals(wordList.size(), 2);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user