RED-10290: Improve SearchImplementation logic for dictionaries

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java

@@ -15,6 +15,8 @@ import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
 
+import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.utils.Patterns;
@@ -22,11 +24,13 @@ import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundExcepti
 
 import lombok.Data;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 
 /**
  * A class representing a dictionary used for redaction processes, containing various dictionary models and their versions.
  */
 @Data
+@NoArgsConstructor
 public class Dictionary {
 
     @Getter
@@ -39,6 +43,8 @@ public class Dictionary {
     @Getter
     private DictionaryVersion version;
 
+    private DictionarySearchImplementation dictionarySearch;
+
 
     public Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version) {
 
@@ -124,6 +130,77 @@ public class Dictionary {
     }
 
 
+    public DictionarySearchImplementation getDictionarySearch() {
+
+        if (dictionarySearch == null) {
+            dictionarySearch = new DictionarySearchImplementation(generateCombinedDictionaryValuesMap());
+        }
+        return dictionarySearch;
+    }
+
+
+    private Map<DictionaryIdentifier, List<String>> generateCombinedDictionaryValuesMap() {
+
+        Map<DictionaryIdentifier, List<String>> combinedValuesMap = new HashMap<>();
+
+        for (DictionaryModel model : dictionaryModels) {
+
+            DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive());
+            DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(),
+                                                                                     EntityType.DICTIONARY_REMOVAL,
+                                                                                     model.isDossierDictionary(),
+                                                                                     !model.isCaseInsensitive());
+            DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(),
+                                                                                    EntityType.FALSE_POSITIVE,
+                                                                                    model.isDossierDictionary(),
+                                                                                    !model.isCaseInsensitive());
+            DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(),
+                                                                                           EntityType.FALSE_RECOMMENDATION,
+                                                                                           model.isDossierDictionary(),
+                                                                                           !model.isCaseInsensitive());
+
+            List<String> entryValues = model.getEntries()
+                    .stream()
+                    .filter(e -> !e.isDeleted())
+                    .map(DictionaryEntry::getValue)
+                    .toList();
+
+            List<String> deletedEntryValues = model.getEntries()
+                    .stream()
+                    .filter(DictionaryEntry::isDeleted)
+                    .map(DictionaryEntry::getValue)
+                    .toList();
+
+            List<String> falsePositiveValues = model.getFalsePositives()
+                    .stream()
+                    .filter(e -> !e.isDeleted())
+                    .map(DictionaryEntry::getValue)
+                    .toList();
+
+            List<String> falseRecommendationValues = model.getFalseRecommendations()
+                    .stream()
+                    .filter(e -> !e.isDeleted())
+                    .map(DictionaryEntry::getValue)
+                    .toList();
+
+            if (!entryValues.isEmpty()) {
+                combinedValuesMap.put(entriesIdentifier, entryValues);
+            }
+            if (!deletedEntryValues.isEmpty()) {
+                combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues);
+            }
+            if (!falsePositiveValues.isEmpty()) {
+                combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues);
+            }
+            if (!falseRecommendationValues.isEmpty()) {
+                combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues);
+            }
+        }
+
+        return combinedValuesMap;
+    }
+
+
     /**
      * Adds a local dictionary entry of a specific type.
      *

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifier.java

@@ -0,0 +1,8 @@
+package com.iqser.red.service.redaction.v1.server.model.dictionary;
+
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+
+public record DictionaryIdentifier(String type, EntityType entityType, boolean dossierDictionaryEntry, boolean caseSensitive) {
+
+}
+

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java

@@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j;
  */
 @Data
 @Slf4j
-public class DictionaryModel implements Serializable {
+public class DictionaryModel implements Cloneable {
 
     private final String type;
     private final int rank;
@@ -33,13 +33,13 @@ public class DictionaryModel implements Serializable {
     private final Set<DictionaryEntryModel> falsePositives;
     private final Set<DictionaryEntryModel> falseRecommendations;
 
-    private transient SearchImplementation entriesSearch;
+    private SearchImplementation entriesSearch;
-    private transient SearchImplementation deletionEntriesSearch;
+    private SearchImplementation deletionEntriesSearch;
-    private transient SearchImplementation falsePositiveSearch;
+    private SearchImplementation falsePositiveSearch;
-    private transient SearchImplementation falseRecommendationsSearch;
+    private SearchImplementation falseRecommendationsSearch;
 
     private final HashMap<String, Set<MatchedRule>> localEntriesWithMatchedRules = new HashMap<>();
-    private transient SearchImplementation localSearch;
+    private SearchImplementation localSearch;
 
 
     /**
@@ -172,4 +172,28 @@ public class DictionaryModel implements Serializable {
         return localEntriesWithMatchedRules.get(cleanedValue);
     }
 
+
+    public SearchImplementation test() {
+
+        return this.entriesSearch;
+    }
+
+    @Override
+    public DictionaryModel clone() {
+        try {
+            DictionaryModel cloned = (DictionaryModel) super.clone();
+
+            cloned.entriesSearch = null;
+            cloned.deletionEntriesSearch = null;
+            cloned.falsePositiveSearch = null;
+            cloned.falseRecommendationsSearch = null;
+            cloned.localSearch = null;
+
+            return cloned;
+
+        } catch (CloneNotSupportedException e) {
+            throw new AssertionError("Cloning not supported", e);
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java

@@ -0,0 +1,144 @@
+package com.iqser.red.service.redaction.v1.server.model.dictionary;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+public class DictionarySearchImplementation {
+
+    private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new HashMap<>();
+    private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new HashMap<>();
+    private final DictionaryTrie caseSensitiveTrie;
+    private final DictionaryTrie caseInsensitiveTrie;
+
+
+    public DictionarySearchImplementation(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
+
+        System.out.println("TEST ---> DictionarySearchImplementation CTOR");
+        for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
+            DictionaryIdentifier identifier = entry.getKey();
+            List<String> values = entry.getValue();
+            if (identifier.caseSensitive()) {
+                caseSensitiveEntries.put(identifier, values);
+            } else {
+                caseInsensitiveEntries.put(identifier, values);
+            }
+        }
+
+        this.caseSensitiveTrie = createTrie(caseSensitiveEntries, false);
+        this.caseInsensitiveTrie = createTrie(caseInsensitiveEntries, true);
+    }
+
+
+    private DictionaryTrie createTrie(Map<DictionaryIdentifier, List<String>> entries, boolean ignoreCase) {
+
+        if (entries.isEmpty()) {
+            return null;
+        }
+        DictionaryTrie.Builder builder = new DictionaryTrie.Builder();
+        if (ignoreCase) {
+            builder.ignoreCase();
+        }
+        entries.forEach((identifier, values) -> {
+            for (String value : values) {
+                builder.addKeyword(value, identifier);
+            }
+        });
+        return builder.build();
+    }
+
+
+    public boolean atLeastOneMatches(String text) {
+
+        if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) {
+            return true;
+        }
+        return !caseInsensitiveEntries.isEmpty() && caseInsensitiveTrie != null && caseInsensitiveTrie.containsMatch(text);
+    }
+
+
+    public List<MatchTextRange> getBoundaries(CharSequence text) {
+
+        List<MatchTextRange> matches = new ArrayList<>();
+        addMatchTextRangesForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text);
+        addMatchTextRangesForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text);
+        return matches;
+    }
+
+
+    private void addMatchTextRangesForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchTextRange> matches, CharSequence text) {
+
+        if (!entries.isEmpty() && trie != null) {
+            matches.addAll(trie.parseText(text)
+                                   .stream()
+                                   .map(r -> new MatchTextRange(r.getPayload(), new TextRange(r.getStart(), r.getEnd() + 1)))
+                                   .toList());
+        }
+    }
+
+
+    public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
+
+        return getBoundaries(textBlock, textBlock.getTextRange());
+    }
+
+
+    public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
+
+        List<MatchTextRange> matches = new ArrayList<>();
+        addMatchTextRangesForTrie(text, region, matches, caseSensitiveEntries, caseSensitiveTrie);
+        addMatchTextRangesForTrie(text, region, matches, caseInsensitiveEntries, caseInsensitiveTrie);
+        return matches.stream();
+    }
+
+
+    private void addMatchTextRangesForTrie(CharSequence text,
+                                           TextRange region,
+                                           List<MatchTextRange> matches,
+                                           Map<DictionaryIdentifier, List<String>> entries,
+                                           DictionaryTrie trie) {
+
+        if (!entries.isEmpty() && trie != null) {
+            CharSequence subSequence = text.subSequence(region.start(), region.end());
+            matches.addAll(trie.parseText(subSequence)
+                                   .stream()
+                                   .map(r -> new MatchTextRange(r.getPayload(), new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1)))
+                                   .toList());
+        }
+    }
+
+
+    public List<MatchPosition> getMatches(String text) {
+
+        List<MatchPosition> matches = new ArrayList<>();
+        addMatchPositionsForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text);
+        addMatchPositionsForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text);
+        return matches;
+    }
+
+
+    private void addMatchPositionsForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchPosition> matches, String text) {
+
+        if (!entries.isEmpty() && trie != null) {
+            matches.addAll(trie.parseText(text)
+                                   .stream()
+                                   .map(r -> new MatchPosition(r.getPayload(), r.getStart(), r.getEnd() + 1))
+                                   .toList());
+        }
+    }
+
+
+    public record MatchTextRange(DictionaryIdentifier identifier, TextRange textRange) {
+
+    }
+
+    public record MatchPosition(DictionaryIdentifier identifier, int startIndex, int endIndex) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryTrie.java

@@ -0,0 +1,51 @@
+package com.iqser.red.service.redaction.v1.server.model.dictionary;
+
+import org.ahocorasick.trie.PayloadEmit;
+import org.ahocorasick.trie.PayloadTrie;
+
+import java.util.Collection;
+
+public class DictionaryTrie {
+    private final PayloadTrie<DictionaryIdentifier> trie;
+
+    private DictionaryTrie(PayloadTrie<DictionaryIdentifier> trie) {
+        this.trie = trie;
+    }
+
+    public static class Builder {
+        private final PayloadTrie.PayloadTrieBuilder<DictionaryIdentifier> builder;
+
+        public Builder() {
+            this.builder = PayloadTrie.builder();
+        }
+
+        public Builder ignoreCase() {
+            builder.ignoreCase();
+            return this;
+        }
+
+        public Builder addKeyword(String keyword, DictionaryIdentifier payload) {
+            builder.addKeyword(keyword, payload);
+            return this;
+        }
+
+        public Builder addKeywords(Collection<String> keywords, DictionaryIdentifier payload) {
+            for (String keyword : keywords) {
+                builder.addKeyword(keyword, payload);
+            }
+            return this;
+        }
+
+        public DictionaryTrie build() {
+            return new DictionaryTrie(builder.build());
+        }
+    }
+
+    public Collection<PayloadEmit<DictionaryIdentifier>> parseText(CharSequence text) {
+        return trie.parseText(text);
+    }
+
+    public boolean containsMatch(CharSequence text) {
+        return trie.containsMatch(text);
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -7,7 +7,9 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@@ -39,25 +41,45 @@ public class DictionarySearchService {
     @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
     public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
 
+        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
+        dictionary.getDictionarySearch().getBoundaries(node.getTextBlock())
+                .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary.textRange()))
+                .forEach(match -> {
+
+                    EntityType entityType;
+                    if (dictionary.isHint(match.identifier().type()) && match.identifier().entityType() == EntityType.ENTITY) {
+                        entityType = EntityType.HINT;
+                    } else {
+                        entityType = match.identifier().entityType();
+                    }
+
+                    Set<Engine> engines = match.identifier().dossierDictionaryEntry() ? Set.of(Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY);
+                    entityCreationService.byTextRangeWithEngine(match.textRange(), match.identifier().type(), entityType, node, engines)
+                            .ifPresent(entity -> {
+                                entity.setDictionaryEntry(true);
+                                entity.setDossierDictionaryEntry(match.identifier().dossierDictionaryEntry());
+                                if (entityType.equals(EntityType.DICTIONARY_REMOVAL)) {
+                                    entity.ignore("DICT.0.0", "Ignore Dossier Dictionary Entity with DICTIONARY_REMOVAL entity type");
+                                }
+                            });
+                });
+    }
+
+
+    @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries-old")
+    public void addDictionaryEntitiesOld(Dictionary dictionary, SemanticNode node) {
+
         dictionary.getDictionaryModels()
-                .stream()
-                .parallel()
                 .forEach(model -> {
-                    synchronized (node) {
+                    bySearchImplementationAsDictionary(model.getEntriesSearch(),
-                        bySearchImplementationAsDictionary(model.getEntriesSearch(),
+                                                       model.getType(),
-                                                           model.getType(),
+                                                       model.isHint() ? EntityType.HINT : EntityType.ENTITY,
-                                                           model.isHint() ? EntityType.HINT : EntityType.ENTITY,
+                                                       node,
-                                                           node,
+                                                       model.isDossierDictionary());
-                                                           model.isDossierDictionary());
+                    bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
-                        bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
+                    bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
-                        bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
+                    if (model.isDossierDictionary()) {
-                                                           model.getType(),
+                        bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
-                                                           EntityType.FALSE_RECOMMENDATION,
-                                                           node,
-                                                           model.isDossierDictionary());
-                        if (model.isDossierDictionary()) {
-                            bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
-                        }
                     }
                 });
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -8,20 +8,19 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;
 
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.iqser.red.service.dictionarymerge.commons.CommonsDictionaryModel;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
-import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
@@ -54,13 +53,12 @@ public class DictionaryService {
 
     private final RedactionServiceSettings settings;
 
-    private final DictionaryMergeService dictionaryMergeService;
-
     private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
+    private ConcurrentMap<String, Dictionary> dictionaryCache = new ConcurrentHashMap<>();
 
 
     @PostConstruct
-    protected void createCache() {
+    protected void createCaches() {
 
         tenantDictionaryCache = CacheBuilder.newBuilder()
                 .maximumSize(settings.getDictionaryCacheMaximumSize())
@@ -71,6 +69,7 @@ public class DictionaryService {
                         return new TenantDictionary();
                     }
                 });
+
     }
 
 
@@ -275,15 +274,16 @@ public class DictionaryService {
                             falsePositives.addAll(newEntries.getFalsePositives());
                             falseRecommendations.addAll(newEntries.getFalseRecommendations());
 
-                            return new DictionaryModel(t.getType(),
+                            DictionaryModel dictionaryModel = new DictionaryModel(t.getType(),
-                                                       t.getRank(),
+                                                                                  t.getRank(),
-                                                       convertColor(t.getHexColor()),
+                                                                                  convertColor(t.getHexColor()),
-                                                       t.isCaseInsensitive(),
+                                                                                  t.isCaseInsensitive(),
-                                                       t.isHint(),
+                                                                                  t.isHint(),
-                                                       entries,
+                                                                                  entries,
-                                                       falsePositives,
+                                                                                  falsePositives,
-                                                       falseRecommendations,
+                                                                                  falseRecommendations,
-                                                       dossierId != null);
+                                                                                  dossierId != null);
+                            return dictionaryModel;
                         })
                         .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                         .collect(Collectors.toList());
@@ -382,31 +382,54 @@ public class DictionaryService {
 
         DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
         List<DictionaryModel> dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
-        dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
+        dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone()));
 
-        // add dossier
+        // Add dossier
         long dossierDictionaryVersion = -1;
         if (dossierDictionaryExists(dossierId)) {
             DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId);
             List<DictionaryModel> dossierDictionaries = dossierRepresentation.getDictionary();
-            dossierDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
+            dossierDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone()));
             return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierRepresentation.getDictionaryVersion());
         } else {
             return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierDictionaryVersion);
         }
-
     }
 
 
     private Dictionary getDictionary(List<DictionaryModel> mergedDictionaries, DictionaryRepresentation dossierTemplateRepresentation, long dossierDictionaryVersion) {
 
-        return new Dictionary(mergedDictionaries.stream()
+        // todo: we need caching here ? --> do not create a new one always, so we do not have as many creations of the searchImplementation
-                                      .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
+        // or we add it to DictionaryRepresentation and add it here as well?
-                                      .collect(Collectors.toList()),
+        // maybe redis as well?
-                              DictionaryVersion.builder()
+        System.out.println("TEST ---> getDictionary");
-                                      .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
+        DictionaryVersion dictionaryVersion = DictionaryVersion.builder()
-                                      .dossierVersion(dossierDictionaryVersion)
+                .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
-                                      .build());
+                .dossierVersion(dossierDictionaryVersion)
+                .build();
+        String tenantId = TenantContext.getTenantId();
+
+        Dictionary cachedDictionary = dictionaryCache.get(tenantId);
+
+        if (cachedDictionary != null) {
+            DictionaryVersion cachedVersion = cachedDictionary.getVersion();
+
+            boolean isUpToDate = (dictionaryVersion.getDossierTemplateVersion() <= cachedVersion.getDossierTemplateVersion())
+                                 && (dictionaryVersion.getDossierVersion() <= cachedVersion.getDossierVersion());
+
+            if (isUpToDate) {
+                System.out.println("TEST ---> cached Dictionary ");
+                return cachedDictionary;
+            }
+        }
+
+        Dictionary dictionary = new Dictionary(mergedDictionaries.stream()
+                                                       .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
+                                                       .collect(Collectors.toList()), dictionaryVersion);
+        dictionaryCache.put(tenantId, dictionary);
+
+        return dictionary;
+
     }
 
 
@@ -463,38 +486,4 @@ public class DictionaryService {
         }
     }
 
-
-    private List<CommonsDictionaryModel> convertDictionaryModel(List<DictionaryModel> dictionaries) {
-
-        return dictionaries.stream()
-                .map(d -> CommonsDictionaryModel.builder()
-                        .type(d.getType())
-                        .rank(d.getRank())
-                        .color(d.getColor())
-                        .caseInsensitive(d.isCaseInsensitive())
-                        .hint(d.isHint())
-                        .isDossierDictionary(d.isDossierDictionary())
-                        .entries(d.getEntries())
-                        .falsePositives(d.getFalsePositives())
-                        .falseRecommendations(d.getFalseRecommendations())
-                        .build())
-                .collect(Collectors.toList());
-    }
-
-
-    private List<DictionaryModel> convertCommonsDictionaryModel(List<CommonsDictionaryModel> commonsDictionaries) {
-
-        return commonsDictionaries.stream()
-                .map(cd -> new DictionaryModel(cd.getType(),
-                                               cd.getRank(),
-                                               cd.getColor(),
-                                               cd.isCaseInsensitive(),
-                                               cd.isHint(),
-                                               cd.getEntries(),
-                                               cd.getFalsePositives(),
-                                               cd.getFalseRecommendations(),
-                                               cd.isDossierDictionary()))
-                .collect(Collectors.toList());
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java

@@ -0,0 +1,114 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static org.springframework.test.util.AssertionErrors.assertEquals;
+
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+
+@Disabled
+public class DictionaryPerformanceTest {
+
+    private static final int LARGE_DICTIONARY_SIZE = 6_000;
+    private static final int LARGE_TEXT_REPETITIONS = 500_000;
+    private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
+                                                    + "Entity_1 match text. Recommendation_1 also here. Random text continues. ";
+
+
+    @Test
+    public void testLargeFileWithLargeDictionaries() {
+
+        Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
+
+        IntStream.range(0, 6)
+                .forEach(i -> {
+                    EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION;
+                    boolean caseSensitive = i % 2 == 0;
+
+                    DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive);
+                    List<String> dictionary = generateLargeDictionary();
+
+                    // Add specific terms that are included in the large text for matches
+                    if (i == 0) {
+                        dictionary.add("Entity_1");
+                    }
+                    if (i == 1) {
+                        dictionary.add("recommendation_1");
+                    }
+                    dictionaryValues.put(identifier, dictionary);
+                });
+
+        long dictionaryTrieConstructionStart = System.currentTimeMillis();
+        DictionarySearchImplementation dictionarySearchImpl = new DictionarySearchImplementation(dictionaryValues);
+        long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart;
+
+        long searchTrieConstructionStart = System.currentTimeMillis();
+        List<SearchImplementation> searchImplementations = dictionaryValues.entrySet()
+                .stream()
+                .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive()))
+                .toList();
+        long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart;
+
+        String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
+
+        long dictionarySearchStart = System.currentTimeMillis();
+        List<DictionarySearchImplementation.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundaries(largeText);
+        long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart;
+
+        long searchImplStart = System.currentTimeMillis();
+        List<TextRange> searchMatches = new ArrayList<>();
+        for (SearchImplementation searchImpl : searchImplementations) {
+            searchMatches.addAll(searchImpl.getBoundaries(largeText));
+        }
+        long searchImplDuration = System.currentTimeMillis() - searchImplStart;
+
+        System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration);
+        System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration, dictionaryMatches.size());
+        System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration);
+        System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration, searchMatches.size());
+
+        assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() : "Both implementations should find entities.";
+        assertEquals("Both implementations should find the same number of matches", dictionaryMatches.size(), searchMatches.size());
+    }
+
+
+    private List<String> generateLargeDictionary() {
+
+        return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString())
+                .collect(Collectors.toList());
+    }
+
+
+    static final class RandomStringGenerator {
+
+        private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+        private static final int STRING_LENGTH = 50;
+        private static final SecureRandom RANDOM = new SecureRandom();
+
+
+        public static String generateRandomString() {
+
+            StringBuilder sb = new StringBuilder(STRING_LENGTH);
+            for (int i = 0; i < STRING_LENGTH; i++) {
+                int index = RANDOM.nextInt(CHARACTERS.length());
+                sb.append(CHARACTERS.charAt(index));
+            }
+            return sb.toString();
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java

@@ -36,9 +36,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
 import com.iqser.red.service.redaction.v1.server.logger.Context;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
@@ -131,11 +129,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
 
         long dictionarySearchStart = System.currentTimeMillis();
         List<TextEntity> foundEntities = new LinkedList<>();
-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+        findEntitiesWithSearchImplementation(document, dictionary.getDictionarySearch(), foundEntities);
-            findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
-            findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
-            findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
-        }
         System.out.printf("Dictionary search took %d ms and found %d entities\n", System.currentTimeMillis() - dictionarySearchStart, foundEntities.size());
 
         long graphInsertionStart = System.currentTimeMillis();
@@ -218,11 +212,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
             totalGraphTime += graphTime;
 
             var searchStart = System.currentTimeMillis();
-            for (var model : dictionary.getDictionaryModels()) {
+            findEntitiesWithSearchImplementation(document, dictionary.getDictionarySearch(), foundEntities);
-                findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
-                findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
-                findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
-            }
             var searchTime = System.currentTimeMillis() - searchStart;
             totalSearchTime += searchTime;
 
@@ -300,16 +290,12 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
     }
 
 
-    private void findEntitiesWithSearchImplementation(Document document,
+    private void findEntitiesWithSearchImplementation(Document document, DictionarySearchImplementation dictionarySearchImplementation, List<TextEntity> foundEntities) {
-                                                      SearchImplementation searchImplementation,
-                                                      EntityType entityType,
-                                                      List<TextEntity> foundEntities,
-                                                      String type) {
 
         TextBlock textBlock = document.getTextBlock();
-        searchImplementation.getBoundaries(textBlock)
+        dictionarySearchImplementation.getBoundaries(textBlock)
-                .filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary))
+                .filter(match -> boundaryIsSurroundedBySeparators(textBlock, match.textRange()))
-                .map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType, document))
+                .map(match -> TextEntity.initialEntityNode(match.textRange(), match.identifier().type(), match.identifier().entityType(), document))
                 .forEach(foundEntities::add);
     }