RED-10290: Improve SearchImplementation logic for dictionaries
This commit is contained in:
maverickstuder
parent
69f7d688d0
commit
62aa7d9187
@ -15,6 +15,8 @@ import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.Patterns;
|
||||
@ -22,11 +24,13 @@ import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundExcepti
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* A class representing a dictionary used for redaction processes, containing various dictionary models and their versions.
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class Dictionary {
|
||||
|
||||
@Getter
|
||||
@ -39,6 +43,8 @@ public class Dictionary {
|
||||
@Getter
|
||||
private DictionaryVersion version;
|
||||
|
||||
private DictionarySearchImplementation dictionarySearch;
|
||||
|
||||
|
||||
public Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version) {
|
||||
|
||||
@ -124,6 +130,77 @@ public class Dictionary {
|
||||
}
|
||||
|
||||
|
||||
public DictionarySearchImplementation getDictionarySearch() {
|
||||
|
||||
if (dictionarySearch == null) {
|
||||
dictionarySearch = new DictionarySearchImplementation(generateCombinedDictionaryValuesMap());
|
||||
}
|
||||
return dictionarySearch;
|
||||
}
|
||||
|
||||
|
||||
private Map<DictionaryIdentifier, List<String>> generateCombinedDictionaryValuesMap() {
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> combinedValuesMap = new HashMap<>();
|
||||
|
||||
for (DictionaryModel model : dictionaryModels) {
|
||||
|
||||
DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive());
|
||||
DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(),
|
||||
EntityType.DICTIONARY_REMOVAL,
|
||||
model.isDossierDictionary(),
|
||||
!model.isCaseInsensitive());
|
||||
DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(),
|
||||
EntityType.FALSE_POSITIVE,
|
||||
model.isDossierDictionary(),
|
||||
!model.isCaseInsensitive());
|
||||
DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(),
|
||||
EntityType.FALSE_RECOMMENDATION,
|
||||
model.isDossierDictionary(),
|
||||
!model.isCaseInsensitive());
|
||||
|
||||
List<String> entryValues = model.getEntries()
|
||||
.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
|
||||
List<String> deletedEntryValues = model.getEntries()
|
||||
.stream()
|
||||
.filter(DictionaryEntry::isDeleted)
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
|
||||
List<String> falsePositiveValues = model.getFalsePositives()
|
||||
.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
|
||||
List<String> falseRecommendationValues = model.getFalseRecommendations()
|
||||
.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.toList();
|
||||
|
||||
if (!entryValues.isEmpty()) {
|
||||
combinedValuesMap.put(entriesIdentifier, entryValues);
|
||||
}
|
||||
if (!deletedEntryValues.isEmpty()) {
|
||||
combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues);
|
||||
}
|
||||
if (!falsePositiveValues.isEmpty()) {
|
||||
combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues);
|
||||
}
|
||||
if (!falseRecommendationValues.isEmpty()) {
|
||||
combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues);
|
||||
}
|
||||
}
|
||||
|
||||
return combinedValuesMap;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Adds a local dictionary entry of a specific type.
|
||||
*
|
||||
|
||||
@ -0,0 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
|
||||
public record DictionaryIdentifier(String type, EntityType entityType, boolean dossierDictionaryEntry, boolean caseSensitive) {
|
||||
|
||||
}
|
||||
|
||||
@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
*/
|
||||
@Data
|
||||
@Slf4j
|
||||
public class DictionaryModel implements Serializable {
|
||||
public class DictionaryModel implements Cloneable {
|
||||
|
||||
private final String type;
|
||||
private final int rank;
|
||||
@ -33,13 +33,13 @@ public class DictionaryModel implements Serializable {
|
||||
private final Set<DictionaryEntryModel> falsePositives;
|
||||
private final Set<DictionaryEntryModel> falseRecommendations;
|
||||
|
||||
private transient SearchImplementation entriesSearch;
|
||||
private transient SearchImplementation deletionEntriesSearch;
|
||||
private transient SearchImplementation falsePositiveSearch;
|
||||
private transient SearchImplementation falseRecommendationsSearch;
|
||||
private SearchImplementation entriesSearch;
|
||||
private SearchImplementation deletionEntriesSearch;
|
||||
private SearchImplementation falsePositiveSearch;
|
||||
private SearchImplementation falseRecommendationsSearch;
|
||||
|
||||
private final HashMap<String, Set<MatchedRule>> localEntriesWithMatchedRules = new HashMap<>();
|
||||
private transient SearchImplementation localSearch;
|
||||
private SearchImplementation localSearch;
|
||||
|
||||
|
||||
/**
|
||||
@ -172,4 +172,28 @@ public class DictionaryModel implements Serializable {
|
||||
return localEntriesWithMatchedRules.get(cleanedValue);
|
||||
}
|
||||
|
||||
|
||||
public SearchImplementation test() {
|
||||
|
||||
return this.entriesSearch;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DictionaryModel clone() {
|
||||
try {
|
||||
DictionaryModel cloned = (DictionaryModel) super.clone();
|
||||
|
||||
cloned.entriesSearch = null;
|
||||
cloned.deletionEntriesSearch = null;
|
||||
cloned.falsePositiveSearch = null;
|
||||
cloned.falseRecommendationsSearch = null;
|
||||
cloned.localSearch = null;
|
||||
|
||||
return cloned;
|
||||
|
||||
} catch (CloneNotSupportedException e) {
|
||||
throw new AssertionError("Cloning not supported", e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,144 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
|
||||
public class DictionarySearchImplementation {
|
||||
|
||||
private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new HashMap<>();
|
||||
private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new HashMap<>();
|
||||
private final DictionaryTrie caseSensitiveTrie;
|
||||
private final DictionaryTrie caseInsensitiveTrie;
|
||||
|
||||
|
||||
public DictionarySearchImplementation(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
|
||||
|
||||
System.out.println("TEST ---> DictionarySearchImplementation CTOR");
|
||||
for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
|
||||
DictionaryIdentifier identifier = entry.getKey();
|
||||
List<String> values = entry.getValue();
|
||||
if (identifier.caseSensitive()) {
|
||||
caseSensitiveEntries.put(identifier, values);
|
||||
} else {
|
||||
caseInsensitiveEntries.put(identifier, values);
|
||||
}
|
||||
}
|
||||
|
||||
this.caseSensitiveTrie = createTrie(caseSensitiveEntries, false);
|
||||
this.caseInsensitiveTrie = createTrie(caseInsensitiveEntries, true);
|
||||
}
|
||||
|
||||
|
||||
private DictionaryTrie createTrie(Map<DictionaryIdentifier, List<String>> entries, boolean ignoreCase) {
|
||||
|
||||
if (entries.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
DictionaryTrie.Builder builder = new DictionaryTrie.Builder();
|
||||
if (ignoreCase) {
|
||||
builder.ignoreCase();
|
||||
}
|
||||
entries.forEach((identifier, values) -> {
|
||||
for (String value : values) {
|
||||
builder.addKeyword(value, identifier);
|
||||
}
|
||||
});
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
|
||||
public boolean atLeastOneMatches(String text) {
|
||||
|
||||
if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) {
|
||||
return true;
|
||||
}
|
||||
return !caseInsensitiveEntries.isEmpty() && caseInsensitiveTrie != null && caseInsensitiveTrie.containsMatch(text);
|
||||
}
|
||||
|
||||
|
||||
public List<MatchTextRange> getBoundaries(CharSequence text) {
|
||||
|
||||
List<MatchTextRange> matches = new ArrayList<>();
|
||||
addMatchTextRangesForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text);
|
||||
addMatchTextRangesForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text);
|
||||
return matches;
|
||||
}
|
||||
|
||||
|
||||
private void addMatchTextRangesForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchTextRange> matches, CharSequence text) {
|
||||
|
||||
if (!entries.isEmpty() && trie != null) {
|
||||
matches.addAll(trie.parseText(text)
|
||||
.stream()
|
||||
.map(r -> new MatchTextRange(r.getPayload(), new TextRange(r.getStart(), r.getEnd() + 1)))
|
||||
.toList());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
|
||||
|
||||
return getBoundaries(textBlock, textBlock.getTextRange());
|
||||
}
|
||||
|
||||
|
||||
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
|
||||
|
||||
List<MatchTextRange> matches = new ArrayList<>();
|
||||
addMatchTextRangesForTrie(text, region, matches, caseSensitiveEntries, caseSensitiveTrie);
|
||||
addMatchTextRangesForTrie(text, region, matches, caseInsensitiveEntries, caseInsensitiveTrie);
|
||||
return matches.stream();
|
||||
}
|
||||
|
||||
|
||||
private void addMatchTextRangesForTrie(CharSequence text,
|
||||
TextRange region,
|
||||
List<MatchTextRange> matches,
|
||||
Map<DictionaryIdentifier, List<String>> entries,
|
||||
DictionaryTrie trie) {
|
||||
|
||||
if (!entries.isEmpty() && trie != null) {
|
||||
CharSequence subSequence = text.subSequence(region.start(), region.end());
|
||||
matches.addAll(trie.parseText(subSequence)
|
||||
.stream()
|
||||
.map(r -> new MatchTextRange(r.getPayload(), new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1)))
|
||||
.toList());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public List<MatchPosition> getMatches(String text) {
|
||||
|
||||
List<MatchPosition> matches = new ArrayList<>();
|
||||
addMatchPositionsForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text);
|
||||
addMatchPositionsForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text);
|
||||
return matches;
|
||||
}
|
||||
|
||||
|
||||
private void addMatchPositionsForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchPosition> matches, String text) {
|
||||
|
||||
if (!entries.isEmpty() && trie != null) {
|
||||
matches.addAll(trie.parseText(text)
|
||||
.stream()
|
||||
.map(r -> new MatchPosition(r.getPayload(), r.getStart(), r.getEnd() + 1))
|
||||
.toList());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public record MatchTextRange(DictionaryIdentifier identifier, TextRange textRange) {
|
||||
|
||||
}
|
||||
|
||||
public record MatchPosition(DictionaryIdentifier identifier, int startIndex, int endIndex) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
|
||||
import org.ahocorasick.trie.PayloadEmit;
|
||||
import org.ahocorasick.trie.PayloadTrie;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
public class DictionaryTrie {
|
||||
private final PayloadTrie<DictionaryIdentifier> trie;
|
||||
|
||||
private DictionaryTrie(PayloadTrie<DictionaryIdentifier> trie) {
|
||||
this.trie = trie;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private final PayloadTrie.PayloadTrieBuilder<DictionaryIdentifier> builder;
|
||||
|
||||
public Builder() {
|
||||
this.builder = PayloadTrie.builder();
|
||||
}
|
||||
|
||||
public Builder ignoreCase() {
|
||||
builder.ignoreCase();
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder addKeyword(String keyword, DictionaryIdentifier payload) {
|
||||
builder.addKeyword(keyword, payload);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder addKeywords(Collection<String> keywords, DictionaryIdentifier payload) {
|
||||
for (String keyword : keywords) {
|
||||
builder.addKeyword(keyword, payload);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public DictionaryTrie build() {
|
||||
return new DictionaryTrie(builder.build());
|
||||
}
|
||||
}
|
||||
|
||||
public Collection<PayloadEmit<DictionaryIdentifier>> parseText(CharSequence text) {
|
||||
return trie.parseText(text);
|
||||
}
|
||||
|
||||
public boolean containsMatch(CharSequence text) {
|
||||
return trie.containsMatch(text);
|
||||
}
|
||||
}
|
||||
@ -7,7 +7,9 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||
@ -39,25 +41,45 @@ public class DictionarySearchService {
|
||||
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
|
||||
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
|
||||
|
||||
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
|
||||
dictionary.getDictionarySearch().getBoundaries(node.getTextBlock())
|
||||
.filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary.textRange()))
|
||||
.forEach(match -> {
|
||||
|
||||
EntityType entityType;
|
||||
if (dictionary.isHint(match.identifier().type()) && match.identifier().entityType() == EntityType.ENTITY) {
|
||||
entityType = EntityType.HINT;
|
||||
} else {
|
||||
entityType = match.identifier().entityType();
|
||||
}
|
||||
|
||||
Set<Engine> engines = match.identifier().dossierDictionaryEntry() ? Set.of(Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY);
|
||||
entityCreationService.byTextRangeWithEngine(match.textRange(), match.identifier().type(), entityType, node, engines)
|
||||
.ifPresent(entity -> {
|
||||
entity.setDictionaryEntry(true);
|
||||
entity.setDossierDictionaryEntry(match.identifier().dossierDictionaryEntry());
|
||||
if (entityType.equals(EntityType.DICTIONARY_REMOVAL)) {
|
||||
entity.ignore("DICT.0.0", "Ignore Dossier Dictionary Entity with DICTIONARY_REMOVAL entity type");
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries-old")
|
||||
public void addDictionaryEntitiesOld(Dictionary dictionary, SemanticNode node) {
|
||||
|
||||
dictionary.getDictionaryModels()
|
||||
.stream()
|
||||
.parallel()
|
||||
.forEach(model -> {
|
||||
synchronized (node) {
|
||||
bySearchImplementationAsDictionary(model.getEntriesSearch(),
|
||||
model.getType(),
|
||||
model.isHint() ? EntityType.HINT : EntityType.ENTITY,
|
||||
node,
|
||||
model.isDossierDictionary());
|
||||
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
|
||||
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
|
||||
model.getType(),
|
||||
EntityType.FALSE_RECOMMENDATION,
|
||||
node,
|
||||
model.isDossierDictionary());
|
||||
if (model.isDossierDictionary()) {
|
||||
bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
|
||||
}
|
||||
bySearchImplementationAsDictionary(model.getEntriesSearch(),
|
||||
model.getType(),
|
||||
model.isHint() ? EntityType.HINT : EntityType.ENTITY,
|
||||
node,
|
||||
model.isDossierDictionary());
|
||||
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
|
||||
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
|
||||
if (model.isDossierDictionary()) {
|
||||
bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@ -8,20 +8,19 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.SerializationUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.iqser.red.service.dictionarymerge.commons.CommonsDictionaryModel;
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
|
||||
@ -54,13 +53,12 @@ public class DictionaryService {
|
||||
|
||||
private final RedactionServiceSettings settings;
|
||||
|
||||
private final DictionaryMergeService dictionaryMergeService;
|
||||
|
||||
private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
|
||||
private ConcurrentMap<String, Dictionary> dictionaryCache = new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
@PostConstruct
|
||||
protected void createCache() {
|
||||
protected void createCaches() {
|
||||
|
||||
tenantDictionaryCache = CacheBuilder.newBuilder()
|
||||
.maximumSize(settings.getDictionaryCacheMaximumSize())
|
||||
@ -71,6 +69,7 @@ public class DictionaryService {
|
||||
return new TenantDictionary();
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -275,15 +274,16 @@ public class DictionaryService {
|
||||
falsePositives.addAll(newEntries.getFalsePositives());
|
||||
falseRecommendations.addAll(newEntries.getFalseRecommendations());
|
||||
|
||||
return new DictionaryModel(t.getType(),
|
||||
t.getRank(),
|
||||
convertColor(t.getHexColor()),
|
||||
t.isCaseInsensitive(),
|
||||
t.isHint(),
|
||||
entries,
|
||||
falsePositives,
|
||||
falseRecommendations,
|
||||
dossierId != null);
|
||||
DictionaryModel dictionaryModel = new DictionaryModel(t.getType(),
|
||||
t.getRank(),
|
||||
convertColor(t.getHexColor()),
|
||||
t.isCaseInsensitive(),
|
||||
t.isHint(),
|
||||
entries,
|
||||
falsePositives,
|
||||
falseRecommendations,
|
||||
dossierId != null);
|
||||
return dictionaryModel;
|
||||
})
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList());
|
||||
@ -382,31 +382,54 @@ public class DictionaryService {
|
||||
|
||||
DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
|
||||
List<DictionaryModel> dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
|
||||
dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
|
||||
dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone()));
|
||||
|
||||
// add dossier
|
||||
// Add dossier
|
||||
long dossierDictionaryVersion = -1;
|
||||
if (dossierDictionaryExists(dossierId)) {
|
||||
DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId);
|
||||
List<DictionaryModel> dossierDictionaries = dossierRepresentation.getDictionary();
|
||||
dossierDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
|
||||
dossierDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone()));
|
||||
return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierRepresentation.getDictionaryVersion());
|
||||
} else {
|
||||
return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierDictionaryVersion);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private Dictionary getDictionary(List<DictionaryModel> mergedDictionaries, DictionaryRepresentation dossierTemplateRepresentation, long dossierDictionaryVersion) {
|
||||
|
||||
return new Dictionary(mergedDictionaries.stream()
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList()),
|
||||
DictionaryVersion.builder()
|
||||
.dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
|
||||
.dossierVersion(dossierDictionaryVersion)
|
||||
.build());
|
||||
// todo: we need caching here ? --> do not create a new one always, so we do not have as many creations of the searchImplementation
|
||||
// or we add it to DictionaryRepresentation and add it here as well?
|
||||
// maybe redis as well?
|
||||
System.out.println("TEST ---> getDictionary");
|
||||
DictionaryVersion dictionaryVersion = DictionaryVersion.builder()
|
||||
.dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
|
||||
.dossierVersion(dossierDictionaryVersion)
|
||||
.build();
|
||||
String tenantId = TenantContext.getTenantId();
|
||||
|
||||
Dictionary cachedDictionary = dictionaryCache.get(tenantId);
|
||||
|
||||
if (cachedDictionary != null) {
|
||||
DictionaryVersion cachedVersion = cachedDictionary.getVersion();
|
||||
|
||||
boolean isUpToDate = (dictionaryVersion.getDossierTemplateVersion() <= cachedVersion.getDossierTemplateVersion())
|
||||
&& (dictionaryVersion.getDossierVersion() <= cachedVersion.getDossierVersion());
|
||||
|
||||
if (isUpToDate) {
|
||||
System.out.println("TEST ---> cached Dictionary ");
|
||||
return cachedDictionary;
|
||||
}
|
||||
}
|
||||
|
||||
Dictionary dictionary = new Dictionary(mergedDictionaries.stream()
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList()), dictionaryVersion);
|
||||
dictionaryCache.put(tenantId, dictionary);
|
||||
|
||||
return dictionary;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -463,38 +486,4 @@ public class DictionaryService {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<CommonsDictionaryModel> convertDictionaryModel(List<DictionaryModel> dictionaries) {
|
||||
|
||||
return dictionaries.stream()
|
||||
.map(d -> CommonsDictionaryModel.builder()
|
||||
.type(d.getType())
|
||||
.rank(d.getRank())
|
||||
.color(d.getColor())
|
||||
.caseInsensitive(d.isCaseInsensitive())
|
||||
.hint(d.isHint())
|
||||
.isDossierDictionary(d.isDossierDictionary())
|
||||
.entries(d.getEntries())
|
||||
.falsePositives(d.getFalsePositives())
|
||||
.falseRecommendations(d.getFalseRecommendations())
|
||||
.build())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
private List<DictionaryModel> convertCommonsDictionaryModel(List<CommonsDictionaryModel> commonsDictionaries) {
|
||||
|
||||
return commonsDictionaries.stream()
|
||||
.map(cd -> new DictionaryModel(cd.getType(),
|
||||
cd.getRank(),
|
||||
cd.getColor(),
|
||||
cd.isCaseInsensitive(),
|
||||
cd.isHint(),
|
||||
cd.getEntries(),
|
||||
cd.getFalsePositives(),
|
||||
cd.getFalseRecommendations(),
|
||||
cd.isDossierDictionary()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,114 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
|
||||
import static org.springframework.test.util.AssertionErrors.assertEquals;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
|
||||
@Disabled
|
||||
public class DictionaryPerformanceTest {
|
||||
|
||||
private static final int LARGE_DICTIONARY_SIZE = 6_000;
|
||||
private static final int LARGE_TEXT_REPETITIONS = 500_000;
|
||||
private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
|
||||
+ "Entity_1 match text. Recommendation_1 also here. Random text continues. ";
|
||||
|
||||
|
||||
@Test
|
||||
public void testLargeFileWithLargeDictionaries() {
|
||||
|
||||
Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
|
||||
|
||||
IntStream.range(0, 6)
|
||||
.forEach(i -> {
|
||||
EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION;
|
||||
boolean caseSensitive = i % 2 == 0;
|
||||
|
||||
DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive);
|
||||
List<String> dictionary = generateLargeDictionary();
|
||||
|
||||
// Add specific terms that are included in the large text for matches
|
||||
if (i == 0) {
|
||||
dictionary.add("Entity_1");
|
||||
}
|
||||
if (i == 1) {
|
||||
dictionary.add("recommendation_1");
|
||||
}
|
||||
dictionaryValues.put(identifier, dictionary);
|
||||
});
|
||||
|
||||
long dictionaryTrieConstructionStart = System.currentTimeMillis();
|
||||
DictionarySearchImplementation dictionarySearchImpl = new DictionarySearchImplementation(dictionaryValues);
|
||||
long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart;
|
||||
|
||||
long searchTrieConstructionStart = System.currentTimeMillis();
|
||||
List<SearchImplementation> searchImplementations = dictionaryValues.entrySet()
|
||||
.stream()
|
||||
.map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive()))
|
||||
.toList();
|
||||
long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart;
|
||||
|
||||
String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
|
||||
|
||||
long dictionarySearchStart = System.currentTimeMillis();
|
||||
List<DictionarySearchImplementation.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundaries(largeText);
|
||||
long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart;
|
||||
|
||||
long searchImplStart = System.currentTimeMillis();
|
||||
List<TextRange> searchMatches = new ArrayList<>();
|
||||
for (SearchImplementation searchImpl : searchImplementations) {
|
||||
searchMatches.addAll(searchImpl.getBoundaries(largeText));
|
||||
}
|
||||
long searchImplDuration = System.currentTimeMillis() - searchImplStart;
|
||||
|
||||
System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration);
|
||||
System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration, dictionaryMatches.size());
|
||||
System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration);
|
||||
System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration, searchMatches.size());
|
||||
|
||||
assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() : "Both implementations should find entities.";
|
||||
assertEquals("Both implementations should find the same number of matches", dictionaryMatches.size(), searchMatches.size());
|
||||
}
|
||||
|
||||
|
||||
private List<String> generateLargeDictionary() {
|
||||
|
||||
return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
static final class RandomStringGenerator {
|
||||
|
||||
private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
||||
private static final int STRING_LENGTH = 50;
|
||||
private static final SecureRandom RANDOM = new SecureRandom();
|
||||
|
||||
|
||||
public static String generateRandomString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder(STRING_LENGTH);
|
||||
for (int i = 0; i < STRING_LENGTH; i++) {
|
||||
int index = RANDOM.nextInt(CHARACTERS.length());
|
||||
sb.append(CHARACTERS.charAt(index));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -36,9 +36,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
|
||||
import com.iqser.red.service.redaction.v1.server.logger.Context;
|
||||
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
@ -131,11 +129,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
|
||||
|
||||
long dictionarySearchStart = System.currentTimeMillis();
|
||||
List<TextEntity> foundEntities = new LinkedList<>();
|
||||
for (DictionaryModel model : dictionary.getDictionaryModels()) {
|
||||
findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
|
||||
findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
|
||||
findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
|
||||
}
|
||||
findEntitiesWithSearchImplementation(document, dictionary.getDictionarySearch(), foundEntities);
|
||||
System.out.printf("Dictionary search took %d ms and found %d entities\n", System.currentTimeMillis() - dictionarySearchStart, foundEntities.size());
|
||||
|
||||
long graphInsertionStart = System.currentTimeMillis();
|
||||
@ -218,11 +212,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
|
||||
totalGraphTime += graphTime;
|
||||
|
||||
var searchStart = System.currentTimeMillis();
|
||||
for (var model : dictionary.getDictionaryModels()) {
|
||||
findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
|
||||
findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
|
||||
findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
|
||||
}
|
||||
findEntitiesWithSearchImplementation(document, dictionary.getDictionarySearch(), foundEntities);
|
||||
var searchTime = System.currentTimeMillis() - searchStart;
|
||||
totalSearchTime += searchTime;
|
||||
|
||||
@ -300,16 +290,12 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
|
||||
}
|
||||
|
||||
|
||||
private void findEntitiesWithSearchImplementation(Document document,
|
||||
SearchImplementation searchImplementation,
|
||||
EntityType entityType,
|
||||
List<TextEntity> foundEntities,
|
||||
String type) {
|
||||
private void findEntitiesWithSearchImplementation(Document document, DictionarySearchImplementation dictionarySearchImplementation, List<TextEntity> foundEntities) {
|
||||
|
||||
TextBlock textBlock = document.getTextBlock();
|
||||
searchImplementation.getBoundaries(textBlock)
|
||||
.filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary))
|
||||
.map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType, document))
|
||||
dictionarySearchImplementation.getBoundaries(textBlock)
|
||||
.filter(match -> boundaryIsSurroundedBySeparators(textBlock, match.textRange()))
|
||||
.map(match -> TextEntity.initialEntityNode(match.textRange(), match.identifier().type(), match.identifier().entityType(), document))
|
||||
.forEach(foundEntities::add);
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user