From 62aa7d9187d7467e9c8230aa4ae363e7d29b782b Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Fri, 25 Oct 2024 20:09:35 +0200 Subject: [PATCH] RED-10290: Improve SearchImplementation logic for dictionaries --- .../server/model/dictionary/Dictionary.java | 77 ++++++++++ .../dictionary/DictionaryIdentifier.java | 8 + .../model/dictionary/DictionaryModel.java | 36 ++++- .../DictionarySearchImplementation.java | 144 ++++++++++++++++++ .../model/dictionary/DictionaryTrie.java | 51 +++++++ .../service/DictionarySearchService.java | 56 ++++--- .../v1/server/service/DictionaryService.java | 109 ++++++------- .../graph/DictionaryPerformanceTest.java | 114 ++++++++++++++ .../DocumentPerformanceIntegrationTest.java | 28 +--- 9 files changed, 519 insertions(+), 104 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifier.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryTrie.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index a650d374..7a180994 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -15,6 +15,8 @@ import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.utils.Patterns; @@ -22,11 +24,13 @@ import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundExcepti import lombok.Data; import lombok.Getter; +import lombok.NoArgsConstructor; /** * A class representing a dictionary used for redaction processes, containing various dictionary models and their versions. */ @Data +@NoArgsConstructor public class Dictionary { @Getter @@ -39,6 +43,8 @@ public class Dictionary { @Getter private DictionaryVersion version; + private DictionarySearchImplementation dictionarySearch; + public Dictionary(List dictionaryModels, DictionaryVersion version) { @@ -124,6 +130,77 @@ public class Dictionary { } + public DictionarySearchImplementation getDictionarySearch() { + + if (dictionarySearch == null) { + dictionarySearch = new DictionarySearchImplementation(generateCombinedDictionaryValuesMap()); + } + return dictionarySearch; + } + + + private Map> generateCombinedDictionaryValuesMap() { + + Map> combinedValuesMap = new HashMap<>(); + + for (DictionaryModel model : dictionaryModels) { + + DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive()); + DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(), + EntityType.DICTIONARY_REMOVAL, + model.isDossierDictionary(), + !model.isCaseInsensitive()); + DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(), + EntityType.FALSE_POSITIVE, + model.isDossierDictionary(), + !model.isCaseInsensitive()); + DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(), + EntityType.FALSE_RECOMMENDATION, + model.isDossierDictionary(), + !model.isCaseInsensitive()); + + List entryValues = model.getEntries() + .stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .toList(); + + List deletedEntryValues = model.getEntries() + .stream() + .filter(DictionaryEntry::isDeleted) + .map(DictionaryEntry::getValue) + .toList(); + + List falsePositiveValues = model.getFalsePositives() + .stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .toList(); + + List falseRecommendationValues = model.getFalseRecommendations() + .stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .toList(); + + if (!entryValues.isEmpty()) { + combinedValuesMap.put(entriesIdentifier, entryValues); + } + if (!deletedEntryValues.isEmpty()) { + combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues); + } + if (!falsePositiveValues.isEmpty()) { + combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues); + } + if (!falseRecommendationValues.isEmpty()) { + combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues); + } + } + + return combinedValuesMap; + } + + /** * Adds a local dictionary entry of a specific type. * diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifier.java new file mode 100644 index 00000000..d524c093 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifier.java @@ -0,0 +1,8 @@ +package com.iqser.red.service.redaction.v1.server.model.dictionary; + +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; + +public record DictionaryIdentifier(String type, EntityType entityType, boolean dossierDictionaryEntry, boolean caseSensitive) { + +} + diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java index 30ae00ad..73423549 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java @@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j; */ @Data @Slf4j -public class DictionaryModel implements Serializable { +public class DictionaryModel implements Cloneable { private final String type; private final int rank; @@ -33,13 +33,13 @@ public class DictionaryModel implements Serializable { private final Set falsePositives; private final Set falseRecommendations; - private transient SearchImplementation entriesSearch; - private transient SearchImplementation deletionEntriesSearch; - private transient SearchImplementation falsePositiveSearch; - private transient SearchImplementation falseRecommendationsSearch; + private SearchImplementation entriesSearch; + private SearchImplementation deletionEntriesSearch; + private SearchImplementation falsePositiveSearch; + private SearchImplementation falseRecommendationsSearch; private final HashMap> localEntriesWithMatchedRules = new HashMap<>(); - private transient SearchImplementation localSearch; + private SearchImplementation localSearch; /** @@ -172,4 +172,28 @@ public class DictionaryModel implements Serializable { return localEntriesWithMatchedRules.get(cleanedValue); } + + public SearchImplementation test() { + + return this.entriesSearch; + } + + @Override + public DictionaryModel clone() { + try { + DictionaryModel cloned = (DictionaryModel) super.clone(); + + cloned.entriesSearch = null; + cloned.deletionEntriesSearch = null; + cloned.falsePositiveSearch = null; + cloned.falseRecommendationsSearch = null; + cloned.localSearch = null; + + return cloned; + + } catch (CloneNotSupportedException e) { + throw new AssertionError("Cloning not supported", e); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java new file mode 100644 index 00000000..8cb8f003 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java @@ -0,0 +1,144 @@ +package com.iqser.red.service.redaction.v1.server.model.dictionary; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; + +public class DictionarySearchImplementation { + + private final Map> caseSensitiveEntries = new HashMap<>(); + private final Map> caseInsensitiveEntries = new HashMap<>(); + private final DictionaryTrie caseSensitiveTrie; + private final DictionaryTrie caseInsensitiveTrie; + + + public DictionarySearchImplementation(Map> dictionaryValues) { + + System.out.println("TEST ---> DictionarySearchImplementation CTOR"); + for (Map.Entry> entry : dictionaryValues.entrySet()) { + DictionaryIdentifier identifier = entry.getKey(); + List values = entry.getValue(); + if (identifier.caseSensitive()) { + caseSensitiveEntries.put(identifier, values); + } else { + caseInsensitiveEntries.put(identifier, values); + } + } + + this.caseSensitiveTrie = createTrie(caseSensitiveEntries, false); + this.caseInsensitiveTrie = createTrie(caseInsensitiveEntries, true); + } + + + private DictionaryTrie createTrie(Map> entries, boolean ignoreCase) { + + if (entries.isEmpty()) { + return null; + } + DictionaryTrie.Builder builder = new DictionaryTrie.Builder(); + if (ignoreCase) { + builder.ignoreCase(); + } + entries.forEach((identifier, values) -> { + for (String value : values) { + builder.addKeyword(value, identifier); + } + }); + return builder.build(); + } + + + public boolean atLeastOneMatches(String text) { + + if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) { + return true; + } + return !caseInsensitiveEntries.isEmpty() && caseInsensitiveTrie != null && caseInsensitiveTrie.containsMatch(text); + } + + + public List getBoundaries(CharSequence text) { + + List matches = new ArrayList<>(); + addMatchTextRangesForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text); + addMatchTextRangesForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text); + return matches; + } + + + private void addMatchTextRangesForTrie(Map> entries, DictionaryTrie trie, List matches, CharSequence text) { + + if (!entries.isEmpty() && trie != null) { + matches.addAll(trie.parseText(text) + .stream() + .map(r -> new MatchTextRange(r.getPayload(), new TextRange(r.getStart(), r.getEnd() + 1))) + .toList()); + } + } + + + public Stream getBoundaries(TextBlock textBlock) { + + return getBoundaries(textBlock, textBlock.getTextRange()); + } + + + public Stream getBoundaries(CharSequence text, TextRange region) { + + List matches = new ArrayList<>(); + addMatchTextRangesForTrie(text, region, matches, caseSensitiveEntries, caseSensitiveTrie); + addMatchTextRangesForTrie(text, region, matches, caseInsensitiveEntries, caseInsensitiveTrie); + return matches.stream(); + } + + + private void addMatchTextRangesForTrie(CharSequence text, + TextRange region, + List matches, + Map> entries, + DictionaryTrie trie) { + + if (!entries.isEmpty() && trie != null) { + CharSequence subSequence = text.subSequence(region.start(), region.end()); + matches.addAll(trie.parseText(subSequence) + .stream() + .map(r -> new MatchTextRange(r.getPayload(), new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1))) + .toList()); + } + } + + + public List getMatches(String text) { + + List matches = new ArrayList<>(); + addMatchPositionsForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text); + addMatchPositionsForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text); + return matches; + } + + + private void addMatchPositionsForTrie(Map> entries, DictionaryTrie trie, List matches, String text) { + + if (!entries.isEmpty() && trie != null) { + matches.addAll(trie.parseText(text) + .stream() + .map(r -> new MatchPosition(r.getPayload(), r.getStart(), r.getEnd() + 1)) + .toList()); + } + } + + + public record MatchTextRange(DictionaryIdentifier identifier, TextRange textRange) { + + } + + public record MatchPosition(DictionaryIdentifier identifier, int startIndex, int endIndex) { + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryTrie.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryTrie.java new file mode 100644 index 00000000..a6a5c33e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryTrie.java @@ -0,0 +1,51 @@ +package com.iqser.red.service.redaction.v1.server.model.dictionary; + +import org.ahocorasick.trie.PayloadEmit; +import org.ahocorasick.trie.PayloadTrie; + +import java.util.Collection; + +public class DictionaryTrie { + private final PayloadTrie trie; + + private DictionaryTrie(PayloadTrie trie) { + this.trie = trie; + } + + public static class Builder { + private final PayloadTrie.PayloadTrieBuilder builder; + + public Builder() { + this.builder = PayloadTrie.builder(); + } + + public Builder ignoreCase() { + builder.ignoreCase(); + return this; + } + + public Builder addKeyword(String keyword, DictionaryIdentifier payload) { + builder.addKeyword(keyword, payload); + return this; + } + + public Builder addKeywords(Collection keywords, DictionaryIdentifier payload) { + for (String keyword : keywords) { + builder.addKeyword(keyword, payload); + } + return this; + } + + public DictionaryTrie build() { + return new DictionaryTrie(builder.build()); + } + } + + public Collection> parseText(CharSequence text) { + return trie.parseText(text); + } + + public boolean containsMatch(CharSequence text) { + return trie.containsMatch(text); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index e5a2ab17..4bb6fd16 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -7,7 +7,9 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; @@ -39,25 +41,45 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + dictionary.getDictionarySearch().getBoundaries(node.getTextBlock()) + .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary.textRange())) + .forEach(match -> { + + EntityType entityType; + if (dictionary.isHint(match.identifier().type()) && match.identifier().entityType() == EntityType.ENTITY) { + entityType = EntityType.HINT; + } else { + entityType = match.identifier().entityType(); + } + + Set engines = match.identifier().dossierDictionaryEntry() ? Set.of(Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY); + entityCreationService.byTextRangeWithEngine(match.textRange(), match.identifier().type(), entityType, node, engines) + .ifPresent(entity -> { + entity.setDictionaryEntry(true); + entity.setDossierDictionaryEntry(match.identifier().dossierDictionaryEntry()); + if (entityType.equals(EntityType.DICTIONARY_REMOVAL)) { + entity.ignore("DICT.0.0", "Ignore Dossier Dictionary Entity with DICTIONARY_REMOVAL entity type"); + } + }); + }); + } + + + @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries-old") + public void addDictionaryEntitiesOld(Dictionary dictionary, SemanticNode node) { + dictionary.getDictionaryModels() - .stream() - .parallel() .forEach(model -> { - synchronized (node) { - bySearchImplementationAsDictionary(model.getEntriesSearch(), - model.getType(), - model.isHint() ? EntityType.HINT : EntityType.ENTITY, - node, - model.isDossierDictionary()); - bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); - bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), - model.getType(), - EntityType.FALSE_RECOMMENDATION, - node, - model.isDossierDictionary()); - if (model.isDossierDictionary()) { - bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary()); - } + bySearchImplementationAsDictionary(model.getEntriesSearch(), + model.getType(), + model.isHint() ? EntityType.HINT : EntityType.ENTITY, + node, + model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); + if (model.isDossierDictionary()) { + bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary()); } }); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index c437d4d0..05c323e4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -8,20 +8,19 @@ import java.util.List; import java.util.Locale; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.SerializationUtils; import org.springframework.stereotype.Service; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.iqser.red.service.dictionarymerge.commons.CommonsDictionaryModel; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; -import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; @@ -54,13 +53,12 @@ public class DictionaryService { private final RedactionServiceSettings settings; - private final DictionaryMergeService dictionaryMergeService; - private LoadingCache tenantDictionaryCache; + private ConcurrentMap dictionaryCache = new ConcurrentHashMap<>(); @PostConstruct - protected void createCache() { + protected void createCaches() { tenantDictionaryCache = CacheBuilder.newBuilder() .maximumSize(settings.getDictionaryCacheMaximumSize()) @@ -71,6 +69,7 @@ public class DictionaryService { return new TenantDictionary(); } }); + } @@ -275,15 +274,16 @@ public class DictionaryService { falsePositives.addAll(newEntries.getFalsePositives()); falseRecommendations.addAll(newEntries.getFalseRecommendations()); - return new DictionaryModel(t.getType(), - t.getRank(), - convertColor(t.getHexColor()), - t.isCaseInsensitive(), - t.isHint(), - entries, - falsePositives, - falseRecommendations, - dossierId != null); + DictionaryModel dictionaryModel = new DictionaryModel(t.getType(), + t.getRank(), + convertColor(t.getHexColor()), + t.isCaseInsensitive(), + t.isHint(), + entries, + falsePositives, + falseRecommendations, + dossierId != null); + return dictionaryModel; }) .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) .collect(Collectors.toList()); @@ -382,31 +382,54 @@ public class DictionaryService { DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId); List dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary(); - dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm))); + dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone())); - // add dossier + // Add dossier long dossierDictionaryVersion = -1; if (dossierDictionaryExists(dossierId)) { DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId); List dossierDictionaries = dossierRepresentation.getDictionary(); - dossierDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm))); + dossierDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone())); return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierRepresentation.getDictionaryVersion()); } else { return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierDictionaryVersion); } - } private Dictionary getDictionary(List mergedDictionaries, DictionaryRepresentation dossierTemplateRepresentation, long dossierDictionaryVersion) { - return new Dictionary(mergedDictionaries.stream() - .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) - .collect(Collectors.toList()), - DictionaryVersion.builder() - .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()) - .dossierVersion(dossierDictionaryVersion) - .build()); + // todo: we need caching here ? --> do not create a new one always, so we do not have as many creations of the searchImplementation + // or we add it to DictionaryRepresentation and add it here as well? + // maybe redis as well? + System.out.println("TEST ---> getDictionary"); + DictionaryVersion dictionaryVersion = DictionaryVersion.builder() + .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()) + .dossierVersion(dossierDictionaryVersion) + .build(); + String tenantId = TenantContext.getTenantId(); + + Dictionary cachedDictionary = dictionaryCache.get(tenantId); + + if (cachedDictionary != null) { + DictionaryVersion cachedVersion = cachedDictionary.getVersion(); + + boolean isUpToDate = (dictionaryVersion.getDossierTemplateVersion() <= cachedVersion.getDossierTemplateVersion()) + && (dictionaryVersion.getDossierVersion() <= cachedVersion.getDossierVersion()); + + if (isUpToDate) { + System.out.println("TEST ---> cached Dictionary "); + return cachedDictionary; + } + } + + Dictionary dictionary = new Dictionary(mergedDictionaries.stream() + .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) + .collect(Collectors.toList()), dictionaryVersion); + dictionaryCache.put(tenantId, dictionary); + + return dictionary; + } @@ -463,38 +486,4 @@ public class DictionaryService { } } - - private List convertDictionaryModel(List dictionaries) { - - return dictionaries.stream() - .map(d -> CommonsDictionaryModel.builder() - .type(d.getType()) - .rank(d.getRank()) - .color(d.getColor()) - .caseInsensitive(d.isCaseInsensitive()) - .hint(d.isHint()) - .isDossierDictionary(d.isDossierDictionary()) - .entries(d.getEntries()) - .falsePositives(d.getFalsePositives()) - .falseRecommendations(d.getFalseRecommendations()) - .build()) - .collect(Collectors.toList()); - } - - - private List convertCommonsDictionaryModel(List commonsDictionaries) { - - return commonsDictionaries.stream() - .map(cd -> new DictionaryModel(cd.getType(), - cd.getRank(), - cd.getColor(), - cd.isCaseInsensitive(), - cd.isHint(), - cd.getEntries(), - cd.getFalsePositives(), - cd.getFalseRecommendations(), - cd.isDossierDictionary())) - .collect(Collectors.toList()); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java new file mode 100644 index 00000000..02fbb2ea --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java @@ -0,0 +1,114 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import static org.springframework.test.util.AssertionErrors.assertEquals; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; + +@Disabled +public class DictionaryPerformanceTest { + + private static final int LARGE_DICTIONARY_SIZE = 6_000; + private static final int LARGE_TEXT_REPETITIONS = 500_000; + private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " + + "Entity_1 match text. Recommendation_1 also here. Random text continues. "; + + + @Test + public void testLargeFileWithLargeDictionaries() { + + Map> dictionaryValues = new HashMap<>(); + + IntStream.range(0, 6) + .forEach(i -> { + EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION; + boolean caseSensitive = i % 2 == 0; + + DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive); + List dictionary = generateLargeDictionary(); + + // Add specific terms that are included in the large text for matches + if (i == 0) { + dictionary.add("Entity_1"); + } + if (i == 1) { + dictionary.add("recommendation_1"); + } + dictionaryValues.put(identifier, dictionary); + }); + + long dictionaryTrieConstructionStart = System.currentTimeMillis(); + DictionarySearchImplementation dictionarySearchImpl = new DictionarySearchImplementation(dictionaryValues); + long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart; + + long searchTrieConstructionStart = System.currentTimeMillis(); + List searchImplementations = dictionaryValues.entrySet() + .stream() + .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive())) + .toList(); + long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart; + + String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS); + + long dictionarySearchStart = System.currentTimeMillis(); + List dictionaryMatches = dictionarySearchImpl.getBoundaries(largeText); + long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart; + + long searchImplStart = System.currentTimeMillis(); + List searchMatches = new ArrayList<>(); + for (SearchImplementation searchImpl : searchImplementations) { + searchMatches.addAll(searchImpl.getBoundaries(largeText)); + } + long searchImplDuration = System.currentTimeMillis() - searchImplStart; + + System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration); + System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration, dictionaryMatches.size()); + System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration); + System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration, searchMatches.size()); + + assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() : "Both implementations should find entities."; + assertEquals("Both implementations should find the same number of matches", dictionaryMatches.size(), searchMatches.size()); + } + + + private List generateLargeDictionary() { + + return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString()) + .collect(Collectors.toList()); + } + + + static final class RandomStringGenerator { + + private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + private static final int STRING_LENGTH = 50; + private static final SecureRandom RANDOM = new SecureRandom(); + + + public static String generateRandomString() { + + StringBuilder sb = new StringBuilder(STRING_LENGTH); + for (int i = 0; i < STRING_LENGTH; i++) { + int index = RANDOM.nextInt(CHARACTERS.length()); + sb.append(CHARACTERS.charAt(index)); + } + return sb.toString(); + } + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index bd5a866d..c49e4264 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -36,9 +36,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; @@ -131,11 +129,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration long dictionarySearchStart = System.currentTimeMillis(); List foundEntities = new LinkedList<>(); - for (DictionaryModel model : dictionary.getDictionaryModels()) { - findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType()); - findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType()); - findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType()); - } + findEntitiesWithSearchImplementation(document, dictionary.getDictionarySearch(), foundEntities); System.out.printf("Dictionary search took %d ms and found %d entities\n", System.currentTimeMillis() - dictionarySearchStart, foundEntities.size()); long graphInsertionStart = System.currentTimeMillis(); @@ -218,11 +212,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration totalGraphTime += graphTime; var searchStart = System.currentTimeMillis(); - for (var model : dictionary.getDictionaryModels()) { - findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType()); - findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType()); - findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType()); - } + findEntitiesWithSearchImplementation(document, dictionary.getDictionarySearch(), foundEntities); var searchTime = System.currentTimeMillis() - searchStart; totalSearchTime += searchTime; @@ -300,16 +290,12 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration } - private void findEntitiesWithSearchImplementation(Document document, - SearchImplementation searchImplementation, - EntityType entityType, - List foundEntities, - String type) { + private void findEntitiesWithSearchImplementation(Document document, DictionarySearchImplementation dictionarySearchImplementation, List foundEntities) { TextBlock textBlock = document.getTextBlock(); - searchImplementation.getBoundaries(textBlock) - .filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary)) - .map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType, document)) + dictionarySearchImplementation.getBoundaries(textBlock) + .filter(match -> boundaryIsSurroundedBySeparators(textBlock, match.textRange())) + .map(match -> TextEntity.initialEntityNode(match.textRange(), match.identifier().type(), match.identifier().entityType(), document)) .forEach(foundEntities::add); }