RED-10290: Improve SearchImplementation logic for dictionaries
* use a different implementation as the DoubleArrayTrieDictionarySearch has a massive performance drop as soon as a certain threshold in terms of dictionary size is reached
This commit is contained in:
maverickstuder
parent
a9e4fa31bf
commit
d72e35723c
@ -63,6 +63,7 @@ dependencies {
|
||||
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
|
||||
implementation("org.ahocorasick:ahocorasick:0.9.0")
|
||||
implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2")
|
||||
implementation("com.github.roklenarcic:aho-corasick:1.2")
|
||||
implementation("org.javassist:javassist:3.29.2-GA")
|
||||
|
||||
implementation("org.drools:drools-engine:${droolsVersion}")
|
||||
|
||||
@ -0,0 +1,130 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
|
||||
public abstract class AbstractDictionarySearch implements DictionarySearch {
|
||||
|
||||
protected final Map<String, List<DictionaryIdentifierWithKeyword>> keyWordToIdentifiersMap;
|
||||
|
||||
|
||||
public AbstractDictionarySearch(Map<String, List<DictionaryIdentifierWithKeyword>> keyWordToIdentifiersMap) {
|
||||
|
||||
this.keyWordToIdentifiersMap = keyWordToIdentifiersMap;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text) {
|
||||
|
||||
TextContext textContext = new TextContext(text);
|
||||
return getMatchTextRangeStream(textContext);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
|
||||
|
||||
CharSequence subText = text.subSequence(region.start(), region.end());
|
||||
TextContext textContext = new TextContext(subText, region.start());
|
||||
return getMatchTextRangeStream(textContext);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
|
||||
|
||||
return getBoundaries(textBlock, textBlock.getTextRange());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchPosition> getMatches(String text) {
|
||||
|
||||
TextContext textContext = new TextContext(text);
|
||||
List<MatchPosition> matches = new ArrayList<>();
|
||||
|
||||
parseText(textContext.getLowerText(), (begin, end, value) -> addMatchPositionsForHit(textContext, matches, new Hit(begin, end, value)));
|
||||
|
||||
return matches.stream();
|
||||
}
|
||||
|
||||
|
||||
private Stream<MatchTextRange> getMatchTextRangeStream(TextContext textContext) {
|
||||
|
||||
List<MatchTextRange> matches = new ArrayList<>();
|
||||
|
||||
parseText(textContext.getLowerText(), (begin, end, value) -> addMatchesForHit(textContext, matches, new Hit(begin, end, value)));
|
||||
|
||||
return matches.stream();
|
||||
}
|
||||
|
||||
|
||||
protected abstract void parseText(CharSequence text, HitHandler handler);
|
||||
|
||||
|
||||
protected void addMatchesForHit(TextContext textContext, List<MatchTextRange> matches, Hit hit) {
|
||||
|
||||
int start = textContext.getStart(hit.begin);
|
||||
int end = textContext.getEnd(hit.end);
|
||||
String matchedText = textContext.getMatchedText(hit.begin, hit.end);
|
||||
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
|
||||
|
||||
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
|
||||
if (idkw.identifier().caseSensitive()) {
|
||||
if (matchedText.equals(idkw.keyword())) {
|
||||
matches.add(new MatchTextRange(idkw.identifier(), new TextRange(start, end)));
|
||||
}
|
||||
} else {
|
||||
matches.add(new MatchTextRange(idkw.identifier(), new TextRange(start, end)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void addMatchPositionsForHit(TextContext textContext, List<MatchPosition> matches, Hit hit) {
|
||||
|
||||
int start = textContext.getStart(hit.begin);
|
||||
int end = textContext.getEnd(hit.end);
|
||||
String matchedText = textContext.getMatchedText(hit.begin, hit.end);
|
||||
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
|
||||
|
||||
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
|
||||
MatchPosition matchPosition = new MatchPosition(idkw.identifier(), start, end);
|
||||
if (idkw.identifier().caseSensitive()) {
|
||||
if (matchedText.equals(idkw.keyword())) {
|
||||
matches.add(matchPosition);
|
||||
}
|
||||
} else {
|
||||
matches.add(matchPosition);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected interface HitHandler {
|
||||
|
||||
void handle(int begin, int end, List<DictionaryIdentifierWithKeyword> value);
|
||||
|
||||
}
|
||||
|
||||
protected static class Hit {
|
||||
|
||||
final int begin;
|
||||
final int end;
|
||||
final List<DictionaryIdentifierWithKeyword> value;
|
||||
|
||||
|
||||
Hit(int begin, int end, List<DictionaryIdentifierWithKeyword> value) {
|
||||
|
||||
this.begin = begin;
|
||||
this.end = end;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.roklenarcic.util.strings.AhoCorasickMap;
|
||||
import com.roklenarcic.util.strings.MapMatchListener;
|
||||
import com.roklenarcic.util.strings.StringMap;
|
||||
|
||||
public class AhoCorasickMapDictionarySearch extends AbstractDictionarySearch {
|
||||
|
||||
private final StringMap<List<DictionaryIdentifierWithKeyword>> map;
|
||||
|
||||
|
||||
public AhoCorasickMapDictionarySearch(Map<String, List<DictionaryIdentifierWithKeyword>> keyWordToIdentifiersMap) {
|
||||
|
||||
super(keyWordToIdentifiersMap);
|
||||
map = new AhoCorasickMap<>(keyWordToIdentifiersMap.keySet(), keyWordToIdentifiersMap.values(), false);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void parseText(CharSequence text, HitHandler handler) {
|
||||
|
||||
MapMatchListener<List<DictionaryIdentifierWithKeyword>> listener = (haystack, startPosition, endPosition, value) -> {
|
||||
handler.handle(startPosition, endPosition, value);
|
||||
return true;
|
||||
};
|
||||
map.match(text.toString(), listener);
|
||||
}
|
||||
|
||||
}
|
||||
@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@ -19,57 +20,70 @@ import lombok.SneakyThrows;
|
||||
@RequiredArgsConstructor
|
||||
public class DictionaryFactory {
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public Dictionary create(List<DictionaryModel> dictionaryModels, DictionaryVersion dictionaryVersion) {
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> combinedMap = generateCombinedDictionaryValuesMap(dictionaryModels);
|
||||
DictionarySearch dictionarySearch = new DoubleArrayTrieDictionarySearch(combinedMap);
|
||||
Map<String, List<DictionaryIdentifierWithKeyword>> keyWordToIdentifiersMap = computeStringIdentifiersMap(dictionaryModels);
|
||||
DictionarySearch dictionarySearch = getDictionarySearch(keyWordToIdentifiersMap);
|
||||
|
||||
return new Dictionary(dictionaryModels, dictionaryVersion, dictionarySearch);
|
||||
}
|
||||
|
||||
|
||||
private Map<DictionaryIdentifier, List<String>> generateCombinedDictionaryValuesMap(List<DictionaryModel> dictionaryModels) {
|
||||
private static DictionarySearch getDictionarySearch(Map<String, List<DictionaryIdentifierWithKeyword>> keyWordToIdentifiersMap) {
|
||||
|
||||
// a more sophisticated selection of the dictionarySearch could be done here
|
||||
// but as we do not have the need to fine-tune at the moment we use the all-rounder solution, which is the AhoCoraSickMapDictionarySearch
|
||||
// based on this repository https://github.com/RokLenarcic/AhoCorasick
|
||||
|
||||
// This is an outline how a more complex dictionarySearch decision could be made:
|
||||
// if (!redactionServiceSettings.isPriorityMode() && keyWordToIdentifiersMap.keySet().size() < 50_000) {
|
||||
// dictionarySearch = new DoubleArrayTrieDictionarySearch(keyWordToIdentifiersMap);
|
||||
// } else {
|
||||
// dictionarySearch = new AhoCorasickMapDictionarySearch(keyWordToIdentifiersMap);
|
||||
// }
|
||||
|
||||
return new AhoCorasickMapDictionarySearch(keyWordToIdentifiersMap);
|
||||
}
|
||||
|
||||
|
||||
protected static Map<String, List<DictionaryIdentifierWithKeyword>> computeStringIdentifiersMap(List<DictionaryModel> dictionaryModels) {
|
||||
|
||||
Map<String, List<DictionaryIdentifierWithKeyword>> stringToIdentifiersMap = new HashMap<>();
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> combinedValuesMap = new HashMap<>();
|
||||
for (DictionaryModel model : dictionaryModels) {
|
||||
addDictionaryEntries(combinedValuesMap, model);
|
||||
|
||||
// Add entries for different entity types
|
||||
addEntriesToMap(stringToIdentifiersMap, model, EntityType.ENTITY, model.getEntries(), false);
|
||||
addEntriesToMap(stringToIdentifiersMap, model, EntityType.FALSE_POSITIVE, model.getFalsePositives(), false);
|
||||
addEntriesToMap(stringToIdentifiersMap, model, EntityType.FALSE_RECOMMENDATION, model.getFalseRecommendations(), false);
|
||||
|
||||
if (model.isDossierDictionary()) {
|
||||
addEntriesToMap(stringToIdentifiersMap, model, EntityType.DICTIONARY_REMOVAL, model.getEntries(), true);
|
||||
}
|
||||
}
|
||||
return combinedValuesMap;
|
||||
|
||||
return stringToIdentifiersMap;
|
||||
}
|
||||
|
||||
|
||||
private void addDictionaryEntries(Map<DictionaryIdentifier, List<String>> combinedValuesMap, DictionaryModel model) {
|
||||
private static void addEntriesToMap(Map<String, List<DictionaryIdentifierWithKeyword>> stringToIdentifiersMap,
|
||||
DictionaryModel model,
|
||||
EntityType entityType,
|
||||
Set<DictionaryEntryModel> entries,
|
||||
boolean isDeleted) {
|
||||
|
||||
addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.ENTITY), filterValues(model.getEntries(), false));
|
||||
addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.FALSE_POSITIVE), filterValues(model.getFalsePositives(), false));
|
||||
addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.FALSE_RECOMMENDATION), filterValues(model.getFalseRecommendations(), false));
|
||||
DictionaryIdentifier identifier = new DictionaryIdentifier(model.getType(), entityType, model.isDossierDictionary(), !model.isCaseInsensitive());
|
||||
|
||||
if (model.isDossierDictionary()) {
|
||||
addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.DICTIONARY_REMOVAL), filterValues(model.getEntries(), true));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private DictionaryIdentifier createIdentifier(DictionaryModel model, EntityType entityType) {
|
||||
|
||||
return new DictionaryIdentifier(model.getType(), entityType, model.isDossierDictionary(), !model.isCaseInsensitive());
|
||||
}
|
||||
|
||||
|
||||
private List<String> filterValues(Set<DictionaryEntryModel> entries, boolean isDeleted) {
|
||||
|
||||
return entries.stream()
|
||||
List<String> values = entries.stream()
|
||||
.filter(entry -> entry.isDeleted() == isDeleted)
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
private void addValuesToMap(Map<DictionaryIdentifier, List<String>> map, DictionaryIdentifier key, List<String> values) {
|
||||
|
||||
if (!values.isEmpty()) {
|
||||
map.computeIfAbsent(key, k -> new ArrayList<>()).addAll(values);
|
||||
for (String value : values) {
|
||||
DictionaryIdentifierWithKeyword idWithKeyword = new DictionaryIdentifierWithKeyword(identifier, value);
|
||||
String key = value.toLowerCase(Locale.ROOT);
|
||||
stringToIdentifiersMap.computeIfAbsent(key, k -> new ArrayList<>()).add(idWithKeyword);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
public record DictionaryIdentifierWithKeyword(DictionaryIdentifier identifier, String keyword) {
|
||||
|
||||
}
|
||||
@ -1,177 +1,31 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
public class DoubleArrayTrieDictionarySearch implements DictionarySearch {
|
||||
public class DoubleArrayTrieDictionarySearch extends AbstractDictionarySearch {
|
||||
|
||||
private final AhoCorasickDoubleArrayTrie<List<DictionaryIdentifierWithKeyword>> trie;
|
||||
|
||||
|
||||
public DoubleArrayTrieDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
public DoubleArrayTrieDictionarySearch(Map<String, List<DictionaryIdentifierWithKeyword>> keyWordToIdentifiersMap) {
|
||||
|
||||
super(keyWordToIdentifiersMap);
|
||||
trie = new AhoCorasickDoubleArrayTrie<>();
|
||||
trie.build(computeStringIdentifiersMap(dictionaryValues));
|
||||
trie.build(keyWordToIdentifiersMap);
|
||||
}
|
||||
|
||||
|
||||
private static Map<String, List<DictionaryIdentifierWithKeyword>> computeStringIdentifiersMap(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
|
||||
Map<String, List<DictionaryIdentifierWithKeyword>> stringToIdentifiersMap = new HashMap<>();
|
||||
for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
|
||||
DictionaryIdentifier identifier = entry.getKey();
|
||||
List<String> values = entry.getValue();
|
||||
for (String value : values) {
|
||||
DictionaryIdentifierWithKeyword idWithKeyword = new DictionaryIdentifierWithKeyword(identifier, value);
|
||||
stringToIdentifiersMap.computeIfAbsent(value.toLowerCase(Locale.ROOT), k -> new ArrayList<>()).add(idWithKeyword);
|
||||
}
|
||||
}
|
||||
return stringToIdentifiersMap;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text) {
|
||||
|
||||
TextContext textContext = new TextContext(text);
|
||||
return getMatchTextRangeStream(textContext);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
|
||||
|
||||
CharSequence subText = text.subSequence(region.start(), region.end());
|
||||
TextContext textContext = new TextContext(subText, region.start());
|
||||
return getMatchTextRangeStream(textContext);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
|
||||
|
||||
return getBoundaries(textBlock, textBlock.getTextRange());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Stream<MatchPosition> getMatches(String text) {
|
||||
|
||||
TextContext textContext = new TextContext(text);
|
||||
List<MatchPosition> matches = new ArrayList<>();
|
||||
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(textContext.getLowerText());
|
||||
protected void parseText(CharSequence text, HitHandler handler) {
|
||||
|
||||
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(text);
|
||||
for (AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit : hits) {
|
||||
addMatchPositionsForHit(textContext, matches, hit);
|
||||
handler.handle(hit.begin, hit.end, hit.value);
|
||||
}
|
||||
return matches.stream();
|
||||
}
|
||||
|
||||
|
||||
private Stream<MatchTextRange> getMatchTextRangeStream(TextContext textContext) {
|
||||
|
||||
List<MatchTextRange> matches = new ArrayList<>();
|
||||
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(textContext.getLowerText());
|
||||
|
||||
for (AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit : hits) {
|
||||
addMatchesForHit(textContext, matches, hit);
|
||||
}
|
||||
return matches.stream();
|
||||
}
|
||||
|
||||
|
||||
private void addMatchesForHit(TextContext textContext, List<MatchTextRange> matches, AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit) {
|
||||
|
||||
int start = textContext.getStart(hit.begin);
|
||||
int end = textContext.getEnd(hit.end);
|
||||
String matchedText = textContext.getMatchedText(hit.begin, hit.end);
|
||||
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
|
||||
|
||||
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
|
||||
if (idkw.identifier.caseSensitive()) {
|
||||
if (matchedText.equals(idkw.keyword)) {
|
||||
matches.add(new MatchTextRange(idkw.identifier, new TextRange(start, end)));
|
||||
}
|
||||
} else {
|
||||
matches.add(new MatchTextRange(idkw.identifier, new TextRange(start, end)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void addMatchPositionsForHit(TextContext textContext, List<MatchPosition> matches, AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit) {
|
||||
|
||||
int start = textContext.getStart(hit.begin);
|
||||
int end = textContext.getEnd(hit.end);
|
||||
String matchedText = textContext.getMatchedText(hit.begin, hit.end);
|
||||
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
|
||||
|
||||
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
|
||||
MatchPosition matchPosition = new MatchPosition(idkw.identifier, start, end);
|
||||
if (idkw.identifier.caseSensitive()) {
|
||||
if (matchedText.equals(idkw.keyword)) {
|
||||
matches.add(matchPosition);
|
||||
}
|
||||
} else {
|
||||
matches.add(matchPosition);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static class TextContext {
|
||||
|
||||
private final CharSequence text;
|
||||
@Getter
|
||||
private final String lowerText;
|
||||
private final int offset;
|
||||
|
||||
|
||||
TextContext(CharSequence text, int offset) {
|
||||
|
||||
this.text = text;
|
||||
this.lowerText = text.toString().toLowerCase(Locale.ROOT);
|
||||
this.offset = offset;
|
||||
}
|
||||
|
||||
|
||||
TextContext(CharSequence text) {
|
||||
|
||||
this(text, 0);
|
||||
}
|
||||
|
||||
|
||||
public int getStart(int hitBegin) {
|
||||
|
||||
return hitBegin + offset;
|
||||
}
|
||||
|
||||
|
||||
public int getEnd(int hitEnd) {
|
||||
|
||||
return hitEnd + offset;
|
||||
}
|
||||
|
||||
|
||||
public String getMatchedText(int hitBegin, int hitEnd) {
|
||||
|
||||
return text.subSequence(hitBegin, hitEnd).toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private record DictionaryIdentifierWithKeyword(DictionaryIdentifier identifier, String keyword) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@ import java.util.stream.Stream;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
|
||||
public class TrieDictionarySearch implements DictionarySearch {
|
||||
public class DoubleTrieDictionarySearch implements DictionarySearch {
|
||||
|
||||
private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new HashMap<>();
|
||||
private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new HashMap<>();
|
||||
@ -17,7 +17,7 @@ public class TrieDictionarySearch implements DictionarySearch {
|
||||
private final DictionaryIdentifierTrie caseInsensitiveTrie;
|
||||
|
||||
|
||||
public TrieDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
public DoubleTrieDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
|
||||
for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
|
||||
DictionaryIdentifier identifier = entry.getKey();
|
||||
@ -0,0 +1,46 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
public class TextContext {
|
||||
|
||||
private final CharSequence text;
|
||||
@Getter
|
||||
private final String lowerText;
|
||||
private final int offset;
|
||||
|
||||
|
||||
TextContext(CharSequence text, int offset) {
|
||||
|
||||
this.text = text;
|
||||
this.lowerText = text.toString().toLowerCase(Locale.ROOT);
|
||||
this.offset = offset;
|
||||
}
|
||||
|
||||
|
||||
TextContext(CharSequence text) {
|
||||
|
||||
this(text, 0);
|
||||
}
|
||||
|
||||
|
||||
public int getStart(int hitBegin) {
|
||||
|
||||
return hitBegin + offset;
|
||||
}
|
||||
|
||||
|
||||
public int getEnd(int hitEnd) {
|
||||
|
||||
return hitEnd + offset;
|
||||
}
|
||||
|
||||
|
||||
public String getMatchedText(int hitBegin, int hitEnd) {
|
||||
|
||||
return text.subSequence(hitBegin, hitEnd).toString();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
@ -111,18 +112,16 @@ public class DictionaryService {
|
||||
|
||||
DictionaryVersion latestVersion = updateDictionary(dossierTemplateId, dossierId);
|
||||
|
||||
Set<DictionaryIncrementValue> newValues = new HashSet<>();
|
||||
Set<DictionaryIncrementValue> newValues = Collections.synchronizedSet(new HashSet<>());
|
||||
List<DictionaryModel> templateDictionaries = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId).getDictionary();
|
||||
|
||||
templateDictionaries.forEach(dictionaryModel -> {
|
||||
addNewEntries(dictionaryModel, fromVersion.getDossierTemplateVersion(), newValues);
|
||||
});
|
||||
templateDictionaries.parallelStream()
|
||||
.forEach(dictionaryModel -> addNewEntries(dictionaryModel, fromVersion.getDossierTemplateVersion(), newValues));
|
||||
|
||||
if (dossierDictionaryExists(dossierId)) {
|
||||
List<DictionaryModel> dossierDictionaries = dictionaryCacheService.getDossierDictionary(dossierId).getDictionary();
|
||||
dossierDictionaries.forEach(dictionaryModel -> {
|
||||
addNewEntries(dictionaryModel, fromVersion.getDossierVersion(), newValues);
|
||||
});
|
||||
dossierDictionaries.parallelStream()
|
||||
.forEach(dictionaryModel -> addNewEntries(dictionaryModel, fromVersion.getDossierVersion(), newValues));
|
||||
}
|
||||
|
||||
return new DictionaryIncrement(newValues, latestVersion);
|
||||
@ -247,76 +246,103 @@ public class DictionaryService {
|
||||
Set<DictionaryEntryModel> combinedFalseRecommendations) {
|
||||
|
||||
if (oldModel.isCaseInsensitive() && !newType.isCaseInsensitive()) {
|
||||
// Compute new entries' values in lowercase once
|
||||
Set<String> newEntryValuesLower = newEntries.getEntries()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedEntries.addAll(oldModel.getEntries()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getEntries()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet()).contains(f.getValue()))
|
||||
.filter(f -> !newEntryValuesLower.contains(f.getValue()))
|
||||
.collect(Collectors.toSet()));
|
||||
|
||||
// Similarly for false positives
|
||||
Set<String> newFalsePositivesValuesLower = newEntries.getFalsePositives()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedFalsePositives.addAll(oldModel.getFalsePositives()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getFalsePositives()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet()).contains(f.getValue()))
|
||||
.filter(f -> !newFalsePositivesValuesLower.contains(f.getValue()))
|
||||
.collect(Collectors.toSet()));
|
||||
|
||||
// Similarly for false recommendations
|
||||
Set<String> newFalseRecommendationsValuesLower = newEntries.getFalseRecommendations()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getFalseRecommendations()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet()).contains(f.getValue()))
|
||||
.filter(f -> !newFalseRecommendationsValuesLower.contains(f.getValue()))
|
||||
.collect(Collectors.toSet()));
|
||||
} else if (!oldModel.isCaseInsensitive() && newType.isCaseInsensitive()) {
|
||||
// Compute new entries' values in lowercase once
|
||||
Set<String> newEntryValuesLower = newEntries.getEntries()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedEntries.addAll(oldModel.getEntries()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getEntries()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT)))
|
||||
.filter(f -> !newEntryValuesLower.contains(f.getValue().toLowerCase(Locale.ROOT)))
|
||||
.collect(Collectors.toSet()));
|
||||
|
||||
// Similarly for false positives
|
||||
Set<String> newFalsePositivesValuesLower = newEntries.getFalsePositives()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedFalsePositives.addAll(oldModel.getFalsePositives()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getFalsePositives()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT)))
|
||||
.filter(f -> !newFalsePositivesValuesLower.contains(f.getValue().toLowerCase(Locale.ROOT)))
|
||||
.collect(Collectors.toSet()));
|
||||
|
||||
// Similarly for false recommendations
|
||||
Set<String> newFalseRecommendationsValuesLower = newEntries.getFalseRecommendations()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getFalseRecommendations()
|
||||
.stream()
|
||||
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT)))
|
||||
.filter(f -> !newFalseRecommendationsValuesLower.contains(f.getValue().toLowerCase(Locale.ROOT)))
|
||||
.collect(Collectors.toSet()));
|
||||
} else {
|
||||
// Both have the same case sensitivity
|
||||
Set<String> newEntryValues = newEntries.getEntries()
|
||||
.stream()
|
||||
.map(DictionaryEntryModel::getValue)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedEntries.addAll(oldModel.getEntries()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getEntries()
|
||||
.stream()
|
||||
.map(DictionaryEntryModel::getValue)
|
||||
.collect(Collectors.toSet()).contains(f.getValue()))
|
||||
.filter(f -> !newEntryValues.contains(f.getValue()))
|
||||
.collect(Collectors.toSet()));
|
||||
|
||||
// Similarly for false positives
|
||||
Set<String> newFalsePositivesValues = newEntries.getFalsePositives()
|
||||
.stream()
|
||||
.map(DictionaryEntryModel::getValue)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedFalsePositives.addAll(oldModel.getFalsePositives()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getFalsePositives()
|
||||
.stream()
|
||||
.map(DictionaryEntryModel::getValue)
|
||||
.collect(Collectors.toSet()).contains(f.getValue()))
|
||||
.filter(f -> !newFalsePositivesValues.contains(f.getValue()))
|
||||
.collect(Collectors.toSet()));
|
||||
|
||||
// Similarly for false recommendations
|
||||
Set<String> newFalseRecommendationsValues = newEntries.getFalseRecommendations()
|
||||
.stream()
|
||||
.map(DictionaryEntryModel::getValue)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations()
|
||||
.stream()
|
||||
.filter(f -> !newEntries.getFalseRecommendations()
|
||||
.stream()
|
||||
.map(DictionaryEntryModel::getValue)
|
||||
.collect(Collectors.toSet()).contains(f.getValue()))
|
||||
.filter(f -> !newFalseRecommendationsValues.contains(f.getValue()))
|
||||
.collect(Collectors.toSet()));
|
||||
}
|
||||
}
|
||||
@ -407,8 +433,8 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
return dictionaryFactory.create(mergedDictionaries.stream()
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList()), dictionaryVersion);
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList()), dictionaryVersion);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -2,64 +2,97 @@ package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DoubleArrayTrieDictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.*;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
|
||||
public class DictionarySearchImplementationsTest {
|
||||
|
||||
private static final int LARGE_DICTIONARY_SIZE = 4_000;
|
||||
private static final int LARGE_TEXT_REPETITIONS = 500_000;
|
||||
private static final int LARGE_TEXT_REPETITIONS = 50_000;
|
||||
private static final int MAX_PII_ENTRY_COUNT = 500_000;
|
||||
private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
|
||||
+ "Entity_1 match text. Recommendation_1 also here. Random text continues. ";
|
||||
|
||||
// Dictionary identifiers
|
||||
protected static final String VERTEBRATE_INDICATOR = "vertebrate";
|
||||
protected static final String DICTIONARY_ADDRESS = "CBI_address";
|
||||
protected static final String DICTIONARY_AUTHOR = "CBI_author";
|
||||
protected static final String DICTIONARY_SPONSOR = "CBI_sponsor";
|
||||
protected static final String DICTIONARY_PII = "PII";
|
||||
protected static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
|
||||
protected static final String REDACTION_INDICATOR = "redaction_indicator";
|
||||
protected static final String HINT_ONLY_INDICATOR = "hint_only";
|
||||
protected static final String MUST_REDACT_INDICATOR = "must_redact";
|
||||
protected static final String PUBLISHED_INFORMATION_INDICATOR = "published_information";
|
||||
protected static final String TEST_METHOD_INDICATOR = "test_method";
|
||||
protected static final String PURITY_INDICATOR = "purity";
|
||||
|
||||
|
||||
@Test
|
||||
@Disabled
|
||||
public void performanceTest() {
|
||||
public void performanceTestWithRealDictionaries() {
|
||||
|
||||
// Load dictionaries from files
|
||||
Map<String, List<String>> loadedDictionaries = loadDictionaries();
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
|
||||
Random random = new Random();
|
||||
|
||||
// Generate dictionary values with specific terms for matching
|
||||
IntStream.range(0, 6)
|
||||
.forEach(i -> {
|
||||
EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION;
|
||||
boolean caseSensitive = i % 2 == 0;
|
||||
// Randomly assign EntityType.ENTITY or EntityType.RECOMMENDATION to dictionaries
|
||||
for (Map.Entry<String, List<String>> entry : loadedDictionaries.entrySet()) {
|
||||
String dictionaryName = entry.getKey();
|
||||
List<String> dictionaryTerms = entry.getValue();
|
||||
|
||||
DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive);
|
||||
List<String> dictionary = generateLargeDictionary();
|
||||
EntityType entityType = random.nextBoolean() ? EntityType.ENTITY : EntityType.RECOMMENDATION;
|
||||
boolean caseSensitive = random.nextBoolean();
|
||||
|
||||
// Add specific terms that are included in the large text for matches
|
||||
if (i == 0) {
|
||||
dictionary.add("Entity_1");
|
||||
}
|
||||
if (i == 1) {
|
||||
dictionary.add("recommendation_1");
|
||||
}
|
||||
dictionaryValues.put(identifier, dictionary);
|
||||
});
|
||||
DictionaryIdentifier identifier = new DictionaryIdentifier(
|
||||
dictionaryName,
|
||||
entityType,
|
||||
true,
|
||||
caseSensitive
|
||||
);
|
||||
|
||||
dictionaryValues.put(identifier, dictionaryTerms);
|
||||
}
|
||||
|
||||
// **Added dummy dictionaries as per request**
|
||||
// Case-sensitive dictionary containing "Entity_1"
|
||||
DictionaryIdentifier entity1Identifier = new DictionaryIdentifier(
|
||||
"dummy_case_sensitive",
|
||||
EntityType.ENTITY,
|
||||
true,
|
||||
true // Case-sensitive
|
||||
);
|
||||
dictionaryValues.put(entity1Identifier, List.of("Entity_1"));
|
||||
|
||||
// Case-insensitive dictionary containing "recommendation_1"
|
||||
DictionaryIdentifier recommendation1Identifier = new DictionaryIdentifier(
|
||||
"dummy_case_insensitive",
|
||||
EntityType.RECOMMENDATION,
|
||||
true,
|
||||
false // Case-insensitive
|
||||
);
|
||||
dictionaryValues.put(recommendation1Identifier, List.of("recommendation_1"));
|
||||
|
||||
// Measure construction time for TrieDictionarySearch
|
||||
long trieDictionaryConstructionStart = System.currentTimeMillis();
|
||||
TrieDictionarySearch trieDictionarySearchImpl = new TrieDictionarySearch(dictionaryValues);
|
||||
DoubleTrieDictionarySearch doubleTrieDictionarySearchImpl = new DoubleTrieDictionarySearch(dictionaryValues);
|
||||
long trieDictionaryConstructionDuration = System.currentTimeMillis() - trieDictionaryConstructionStart;
|
||||
|
||||
// Measure construction time for AnotherTrieDictionarySearch
|
||||
long anotherTrieConstructionStart = System.currentTimeMillis();
|
||||
Map<String, List<DictionaryIdentifierWithKeyword>> keyWordToIdentifiersMap = computeStringIdentifiersMap(dictionaryValues);
|
||||
AhoCorasickMapDictionarySearch ahoCorasickMapDictionarySearchImpl = new AhoCorasickMapDictionarySearch(keyWordToIdentifiersMap);
|
||||
long anotherTrieConstructionDuration = System.currentTimeMillis() - anotherTrieConstructionStart;
|
||||
|
||||
// Measure construction time for SearchImplementations
|
||||
long searchTrieConstructionStart = System.currentTimeMillis();
|
||||
List<SearchImplementation> searchImplementations = dictionaryValues.entrySet()
|
||||
@ -70,16 +103,22 @@ public class DictionarySearchImplementationsTest {
|
||||
|
||||
// Measure construction time for DoubleArrayTrieDictionarySearch
|
||||
long doubleArrayTrieConstructionStart = System.currentTimeMillis();
|
||||
DoubleArrayTrieDictionarySearch doubleArrayTrieSearchImpl = new DoubleArrayTrieDictionarySearch(dictionaryValues);
|
||||
DoubleArrayTrieDictionarySearch doubleArrayTrieSearchImpl = new DoubleArrayTrieDictionarySearch(keyWordToIdentifiersMap);
|
||||
long doubleArrayTrieConstructionDuration = System.currentTimeMillis() - doubleArrayTrieConstructionStart;
|
||||
|
||||
|
||||
String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
|
||||
|
||||
// Measure search time for TrieDictionarySearch
|
||||
long trieDictionarySearchStart = System.currentTimeMillis();
|
||||
List<TrieDictionarySearch.MatchTextRange> trieDictionaryMatches = trieDictionarySearchImpl.getBoundariesAsList(largeText);
|
||||
List<DoubleTrieDictionarySearch.MatchTextRange> trieDictionaryMatches = doubleTrieDictionarySearchImpl.getBoundariesAsList(largeText);
|
||||
long trieDictionarySearchDuration = System.currentTimeMillis() - trieDictionarySearchStart;
|
||||
|
||||
// Measure search time for AnotherTrieDictionarySearch
|
||||
long anotherTrieSearchStart = System.currentTimeMillis();
|
||||
List<DictionarySearch.MatchTextRange> anotherTrieMatches = ahoCorasickMapDictionarySearchImpl.getBoundaries(largeText).toList();
|
||||
long anotherTrieSearchDuration = System.currentTimeMillis() - anotherTrieSearchStart;
|
||||
|
||||
// Measure search time for SearchImplementations
|
||||
long searchImplStart = System.currentTimeMillis();
|
||||
List<TextRange> searchMatches = new ArrayList<>();
|
||||
@ -94,72 +133,113 @@ public class DictionarySearchImplementationsTest {
|
||||
long doubleArrayTrieSearchDuration = System.currentTimeMillis() - doubleArrayTrieSearchStart;
|
||||
|
||||
// Output the performance results
|
||||
System.out.printf("TrieDictionarySearch construction took %d ms\n", trieDictionaryConstructionDuration);
|
||||
System.out.printf("TrieDictionarySearch search took %d ms and found %d matches\n", trieDictionarySearchDuration, trieDictionaryMatches.size());
|
||||
System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration);
|
||||
System.out.printf("Combined SearchImplementation search took %d ms and found %d matches\n", searchImplDuration, searchMatches.size());
|
||||
System.out.printf("DoubleArrayTrieDictionarySearch construction took %d ms\n", doubleArrayTrieConstructionDuration);
|
||||
System.out.printf("DoubleArrayTrieDictionarySearch search took %d ms and found %d matches\n", doubleArrayTrieSearchDuration, doubleArrayTrieMatches.size());
|
||||
System.out.println("\nTotal number of keywords is: " + keyWordToIdentifiersMap.size());
|
||||
|
||||
System.out.printf("DoubleTrieDictionarySearch construction took %d ms%n", trieDictionaryConstructionDuration);
|
||||
System.out.printf("DoubleTrieDictionarySearch search took %d ms and found %d matches%n", trieDictionarySearchDuration, trieDictionaryMatches.size());
|
||||
System.out.println();
|
||||
|
||||
System.out.printf("AhoCorasickMapDictionarySearch construction took %d ms%n", anotherTrieConstructionDuration);
|
||||
System.out.printf("AhoCorasickMapDictionarySearch search took %d ms and found %d matches%n", anotherTrieSearchDuration, anotherTrieMatches.size());
|
||||
System.out.println();
|
||||
|
||||
System.out.printf("Multiple Tries construction took %d ms%n", searchTrieConstructionDuration);
|
||||
System.out.printf("Combined SearchImplementations search took %d ms and found %d matches%n", searchImplDuration, searchMatches.size());
|
||||
System.out.println();
|
||||
|
||||
System.out.printf("DoubleArrayTrieDictionarySearch construction took %d ms%n", doubleArrayTrieConstructionDuration);
|
||||
System.out.printf("DoubleArrayTrieDictionarySearch search took %d ms and found %d matches%n", doubleArrayTrieSearchDuration, doubleArrayTrieMatches.size());
|
||||
System.out.println();
|
||||
|
||||
// Assert that all implementations found matches
|
||||
assert !trieDictionaryMatches.isEmpty() && !searchMatches.isEmpty() && !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities.";
|
||||
assert !trieDictionaryMatches.isEmpty() && !anotherTrieMatches.isEmpty() && !searchMatches.isEmpty() && !doubleArrayTrieMatches.isEmpty(): "All implementations should find entities.";
|
||||
|
||||
// Ensure all implementations found the same number of matches
|
||||
assertEquals(trieDictionaryMatches.size(), searchMatches.size(), "Mismatch between TrieDictionarySearch and SearchImplementations");
|
||||
assertEquals(trieDictionaryMatches.size(), doubleArrayTrieMatches.size(), "Mismatch between TrieDictionarySearch and DoubleArrayTrieDictionarySearch");
|
||||
int expectedMatches = trieDictionaryMatches.size();
|
||||
assertEquals(expectedMatches, anotherTrieMatches.size(), "Mismatch between DoubleTrieDictionarySearch and AhoCorasickMapDictionarySearch");
|
||||
assertEquals(expectedMatches, searchMatches.size(), "Mismatch between DoubleTrieDictionarySearch and Combined SearchImplementations");
|
||||
assertEquals(expectedMatches, doubleArrayTrieMatches.size(), "Mismatch between DoubleTrieDictionarySearch and DoubleArrayTrieDictionarySearch");
|
||||
}
|
||||
|
||||
private Map<String, List<String>> loadDictionaries() {
|
||||
Map<String, List<String>> dictionaries = new HashMap<>();
|
||||
|
||||
private List<String> generateLargeDictionary() {
|
||||
dictionaries.put(DICTIONARY_AUTHOR, loadDictionaryFromFile("dictionaries/CBI_author.txt"));
|
||||
dictionaries.put(DICTIONARY_SPONSOR, loadDictionaryFromFile("dictionaries/CBI_sponsor.txt"));
|
||||
dictionaries.put(VERTEBRATE_INDICATOR, loadDictionaryFromFile("dictionaries/vertebrate.txt"));
|
||||
dictionaries.put(DICTIONARY_ADDRESS, loadDictionaryFromFile("dictionaries/CBI_address.txt"));
|
||||
dictionaries.put(NO_REDACTION_INDICATOR, loadDictionaryFromFile("dictionaries/no_redaction_indicator.txt"));
|
||||
dictionaries.put(REDACTION_INDICATOR, loadDictionaryFromFile("dictionaries/redaction_indicator.txt"));
|
||||
dictionaries.put(HINT_ONLY_INDICATOR, loadDictionaryFromFile("dictionaries/hint_only.txt"));
|
||||
dictionaries.put(MUST_REDACT_INDICATOR, loadDictionaryFromFile("dictionaries/must_redact.txt"));
|
||||
dictionaries.put(PUBLISHED_INFORMATION_INDICATOR, loadDictionaryFromFile("dictionaries/published_information.txt"));
|
||||
dictionaries.put(TEST_METHOD_INDICATOR, loadDictionaryFromFile("dictionaries/test_method.txt"));
|
||||
List<String> piis = loadDictionaryFromFile("dictionaries/PII_large.txt");
|
||||
dictionaries.put(DICTIONARY_PII, MAX_PII_ENTRY_COUNT < piis.size() ? piis.subList(0, MAX_PII_ENTRY_COUNT) : piis);
|
||||
dictionaries.put(PURITY_INDICATOR, loadDictionaryFromFile("dictionaries/purity.txt"));
|
||||
|
||||
return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString())
|
||||
.collect(Collectors.toList());
|
||||
return dictionaries;
|
||||
}
|
||||
|
||||
private List<String> loadDictionaryFromFile(String filePath) {
|
||||
List<String> terms = new ArrayList<>();
|
||||
|
||||
static final class RandomStringGenerator {
|
||||
try (BufferedReader reader = new BufferedReader(new InputStreamReader(
|
||||
Objects.requireNonNull(getClass().getClassLoader().getResourceAsStream(filePath))))) {
|
||||
|
||||
private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
||||
private static final int STRING_LENGTH = 50;
|
||||
private static final SecureRandom RANDOM = new SecureRandom();
|
||||
terms = reader.lines()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
|
||||
public static String generateRandomString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder(STRING_LENGTH);
|
||||
for (int i = 0; i < STRING_LENGTH; i++) {
|
||||
int index = RANDOM.nextInt(CHARACTERS.length());
|
||||
sb.append(CHARACTERS.charAt(index));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception e) {
|
||||
System.err.println("Failed to load dictionary from " + filePath + ": " + e.getMessage());
|
||||
}
|
||||
|
||||
return terms;
|
||||
}
|
||||
|
||||
private String cleanDictionaryEntry(String entry) {
|
||||
return entry.trim();
|
||||
}
|
||||
|
||||
|
||||
private static Map<String, List<DictionaryIdentifierWithKeyword>> computeStringIdentifiersMap(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
|
||||
Map<String, List<DictionaryIdentifierWithKeyword>> stringToIdentifiersMap = new HashMap<>();
|
||||
for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
|
||||
DictionaryIdentifier identifier = entry.getKey();
|
||||
List<String> values = entry.getValue();
|
||||
for (String value : values) {
|
||||
DictionaryIdentifierWithKeyword idWithKeyword = new DictionaryIdentifierWithKeyword(identifier, value);
|
||||
stringToIdentifiersMap.computeIfAbsent(value.toLowerCase(Locale.ROOT), k -> new ArrayList<>()).add(idWithKeyword);
|
||||
}
|
||||
}
|
||||
return stringToIdentifiersMap;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiplePayloads() {
|
||||
|
||||
TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(Map.of(new DictionaryIdentifier("type1", EntityType.ENTITY, false, false),
|
||||
List.of("apple", "banana"),
|
||||
new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false),
|
||||
List.of("apple", "orange"),
|
||||
new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false),
|
||||
List.of("apple", "kiwi")));
|
||||
DoubleTrieDictionarySearch dictionarySearchImpl = new DoubleTrieDictionarySearch(Map.of(
|
||||
new DictionaryIdentifier("type1", EntityType.ENTITY, false, false),
|
||||
List.of("apple", "banana"),
|
||||
new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false),
|
||||
List.of("apple", "orange"),
|
||||
new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false),
|
||||
List.of("apple", "kiwi")));
|
||||
|
||||
List<TrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundariesAsList(
|
||||
List<DoubleTrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundariesAsList(
|
||||
"an apple is delicious, a banana and a kiwi as well. orange is a color.");
|
||||
|
||||
assertEquals(dictionaryMatches.size(), 6);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testDoubleArrayTrie() {
|
||||
|
||||
Map<String, List<String>> map = new HashMap<>();
|
||||
String[] keyArray = new String[]{"hers", "his", "she", "he"};
|
||||
String[] keyArray = new String[] { "hers", "his", "she", "he" };
|
||||
for (String key : keyArray) {
|
||||
map.put(key, List.of(key, key, key));
|
||||
}
|
||||
|
||||
@ -29,7 +29,6 @@ import com.iqser.red.service.redaction.v1.server.logger.Context;
|
||||
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user