RED-10290: Improve SearchImplementation logic for dictionaries

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -61,7 +61,8 @@ dependencies {
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
-    implementation("org.ahocorasick:ahocorasick:0.6.3")
+    implementation("org.ahocorasick:ahocorasick:0.9.0")
+    implementation("com.hankcs:aho-corasick-double-array-trie:1.2.2")
     implementation("org.javassist:javassist:3.29.2-GA")
 
     implementation("org.drools:drools-engine:${droolsVersion}")
@@ -87,7 +88,6 @@ dependencies {
     implementation("org.reflections:reflections:0.10.2")
 
     implementation("com.opencsv:opencsv:5.9")
-    implementation("com.gliwka.hyperscan:hyperscan:5.4.11-3.0.0")
     implementation("com.joestelmach:natty:0.13")
 
     testImplementation(project(":rules-management"))

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java

@@ -26,7 +26,7 @@ public class RedactionServiceSettings {
 
     private boolean llmNerServiceEnabled;
 
-    private boolean priorityMode;
+    private boolean priorityMode = false;
 
     private long firstLevelDictionaryCacheMaximumSize = 1000;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java

@@ -2,14 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import static java.lang.String.format;
 
-import java.util.Arrays;
+import java.util.*;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -31,28 +24,70 @@ import lombok.Getter;
 @Data
 public class Dictionary {
 
-    private final Map<String, DictionaryModel> localAccessMap = new HashMap<>();
+    private final Map<String, Map<Level, DictionaryModel>> localAccessMap = new HashMap<>();
 
-    @Getter
     private final DictionaryVersion version;
 
     private final DictionarySearch dictionarySearch;
 
+    public enum Level {
+        DOSSIER_TEMPLATE,
+        DOSSIER
+    }
+
 
     Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version, DictionarySearch dictionarySearch) {
 
-        dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
+        dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), Map.of(getLevel(dm.isDossierDictionary()), dm)));
         this.version = version;
         this.dictionarySearch = dictionarySearch;
     }
 
 
-    public int getDictionaryRank(String type) {
+    private Level getLevel(boolean isDossierDictionary) {
+
+        return isDossierDictionary ? Level.DOSSIER : Level.DOSSIER_TEMPLATE;
+    }
+
+
+    /**
+     * Determines the default level for a given type based on the levels present.
+     * If both levels are present, it defaults to {@code Level.DOSSIER}.
+     *
+     * @param type The type to determine the default level for.
+     * @return The default {@link Level} for the specified type.
+     * @throws NotFoundException If the type is not found in the dictionary.
+     */
+    private Level getDefaultLevel(String type) {
+
+        Map<Level, DictionaryModel> levelMap = localAccessMap.get(type);
+        if (levelMap == null || levelMap.isEmpty()) {
+            throw new NotFoundException("Type: " + type + " is not found");
+        }
+        if (levelMap.containsKey(Level.DOSSIER)) {
+            return Level.DOSSIER;
+        } else {
+            // Use whatever level is present
+            return levelMap.keySet()
+                    .iterator().next();
+        }
+    }
+
+
+    public int getDictionaryRank(String type, Level level) {
 
         if (!localAccessMap.containsKey(type)) {
             return 0;
         }
-        return localAccessMap.get(type).getRank();
+        DictionaryModel model = localAccessMap.get(type)
+                .get(level);
+        return model != null ? model.getRank() : 0;
+    }
+
+
+    public int getDictionaryRank(String type) {
+
+        return getDictionaryRank(type, getDefaultLevel(type));
     }
 
 
@@ -72,6 +107,8 @@ public class Dictionary {
 
         return localAccessMap.values()
                 .stream()
+                .flatMap(levelDictionaryModelMap -> levelDictionaryModelMap.values()
+                        .stream())
                 .toList();
     }
 
@@ -83,56 +120,144 @@ public class Dictionary {
 
 
     /**
-     * Retrieves the {@link DictionaryModel} of a specified type.
+     * Retrieves the {@link DictionaryModel} of a specified type and level.
      *
      * @param type  The type of dictionary model to retrieve.
-     * @return The {@link DictionaryModel} of the specified type.
+     * @param level The level of the dictionary model to retrieve.
-     * @throws NotFoundException If the specified type is not found in the dictionary.
+     * @return The {@link DictionaryModel} of the specified type and level.
+     * @throws NotFoundException If the specified type or level is not found in the dictionary.
      */
-    public DictionaryModel getType(String type) {
+    public DictionaryModel getType(String type, Level level) {
 
-        DictionaryModel model = localAccessMap.get(type);
+        Map<Level, DictionaryModel> levelMap = localAccessMap.get(type);
-        if (model == null) {
+        if (levelMap == null || !levelMap.containsKey(level)) {
-            throw new NotFoundException("Type: " + type + " is not found");
+            throw new NotFoundException("Type: " + type + " with level: " + level + " is not found");
         }
-        return model;
+        return levelMap.get(level);
     }
 
 
     /**
-     * Checks if the dictionary of a specific type is considered a hint.
+     * Retrieves the {@link DictionaryModel} of a specified type at the default level.
+     *
+     * @param type The type of dictionary model to retrieve.
+     * @return The {@link DictionaryModel} of the specified type at the default level.
+     * @throws NotFoundException If the specified type is not found in the dictionary.
+     */
+    public DictionaryModel getType(String type) {
+
+        return getType(type, getDefaultLevel(type));
+    }
+
+
+    /**
+     * Checks if the dictionary of a specific type and level is considered a hint.
+     *
+     * @param type  The type of dictionary to check.
+     * @param level The level of the dictionary to check.
+     * @return true if the dictionary model is marked as a hint, false otherwise.
+     */
+    public boolean isHint(String type, Level level) {
+
+        DictionaryModel model = localAccessMap.get(type)
+                .get(level);
+        return model != null && model.isHint();
+    }
+
+
+    /**
+     * Checks if the dictionary of a specific type is considered a hint at the default level.
      *
      * @param type The type of dictionary to check.
      * @return true if the dictionary model is marked as a hint, false otherwise.
      */
     public boolean isHint(String type) {
 
-        DictionaryModel model = localAccessMap.get(type);
+        return isHint(type, getDefaultLevel(type));
-        if (model != null) {
-            return model.isHint();
-        }
-        return false;
     }
 
 
     /**
-     * Checks if the dictionary of a specific type is case-insensitive.
+     * Checks if the dictionary of a specific type and level is case-insensitive.
+     *
+     * @param type  The type of dictionary to check.
+     * @param level The level of the dictionary to check.
+     * @return true if the dictionary is case-insensitive, false otherwise.
+     */
+    public boolean isCaseInsensitiveDictionary(String type, Level level) {
+
+        DictionaryModel dictionaryModel = localAccessMap.get(type)
+                .get(level);
+        return dictionaryModel != null && dictionaryModel.isCaseInsensitive();
+    }
+
+
+    /**
+     * Checks if the dictionary of a specific type is case-insensitive at the default level.
      *
      * @param type The type of dictionary to check.
      * @return true if the dictionary is case-insensitive, false otherwise.
      */
     public boolean isCaseInsensitiveDictionary(String type) {
 
-        DictionaryModel dictionaryModel = localAccessMap.get(type);
+        return isCaseInsensitiveDictionary(type, getDefaultLevel(type));
-        if (dictionaryModel != null) {
-            return dictionaryModel.isCaseInsensitive();
-        }
-        return false;
     }
 
 
     /**
-     * Adds a local dictionary entry of a specific type.
+     * Adds a local dictionary entry of a specific type and level.
+     *
+     * @param type            The type of dictionary to add the entry to.
+     * @param value           The value of the entry.
+     * @param matchedRules    A collection of {@link MatchedRule} associated with the entry.
+     * @param alsoAddLastname Indicates whether to also add the lastname separately as an entry.
+     * @param level           The level of the dictionary where the entry should be added.
+     * @throws IllegalArgumentException If the specified type does not exist within the dictionary, if the type
+     *                                  does not have any local entries defined, or if the provided value is
+     *                                  blank. This ensures that only valid, non-empty entries
+     *                                  are added to the dictionary.
+     */
+    private void addLocalDictionaryEntry(String type, String value, Collection<MatchedRule> matchedRules, boolean alsoAddLastname, Level level) {
+
+        if (value.isBlank()) {
+            return;
+        }
+        Map<Level, DictionaryModel> levelMap = localAccessMap.get(type);
+        if (levelMap == null || !levelMap.containsKey(level)) {
+            throw new IllegalArgumentException(format("DictionaryModel of type %s with level %s does not exist", type, level));
+        }
+        DictionaryModel dictionaryModel = levelMap.get(level);
+        if (dictionaryModel.getLocalEntriesWithMatchedRules() == null) {
+            throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type));
+        }
+        if (StringUtils.isEmpty(value)) {
+            throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
+        }
+        boolean isCaseInsensitive = dictionaryModel.isCaseInsensitive();
+        Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
+
+        String cleanedValue = value;
+        if (isCaseInsensitive) {
+            cleanedValue = cleanedValue.toLowerCase(Locale.US);
+        }
+        dictionaryModel.getLocalEntriesWithMatchedRules()
+                .merge(cleanedValue.trim(),
+                       matchedRulesSet,
+                       (set1, set2) -> Stream.concat(set1.stream(), set2.stream())
+                               .collect(Collectors.toSet()));
+        if (alsoAddLastname) {
+            String lastname = cleanedValue.split(" ")[0];
+            dictionaryModel.getLocalEntriesWithMatchedRules()
+                    .merge(lastname,
+                           matchedRulesSet,
+                           (set1, set2) -> Stream.concat(set1.stream(), set2.stream())
+                                   .collect(Collectors.toSet()));
+        }
+    }
+
+
+    /**
+     * Adds a local dictionary entry of a specific type at the default level.
      *
      * @param type            The type of dictionary to add the entry to.
      * @param value           The value of the entry.
@@ -145,40 +270,7 @@ public class Dictionary {
      */
     private void addLocalDictionaryEntry(String type, String value, Collection<MatchedRule> matchedRules, boolean alsoAddLastname) {
 
-        if (value.isBlank()) {
+        addLocalDictionaryEntry(type, value, matchedRules, alsoAddLastname, getDefaultLevel(type));
-            return;
-        }
-        if (localAccessMap.get(type) == null) {
-            throw new IllegalArgumentException(format("DictionaryModel of type %s does not exist", type));
-        }
-        if (localAccessMap.get(type).getLocalEntriesWithMatchedRules() == null) {
-            throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type));
-        }
-        if (StringUtils.isEmpty(value)) {
-            throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
-        }
-        boolean isCaseInsensitive = localAccessMap.get(type).isCaseInsensitive();
-        Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
-
-        String cleanedValue = value;
-        if (isCaseInsensitive) {
-            cleanedValue = cleanedValue.toLowerCase(Locale.US);
-        }
-        localAccessMap.get(type)
-                .getLocalEntriesWithMatchedRules()
-                .merge(cleanedValue.trim(),
-                       matchedRulesSet,
-                       (set1, set2) -> Stream.concat(set1.stream(), set2.stream())
-                               .collect(Collectors.toSet()));
-        if (alsoAddLastname) {
-            String lastname = cleanedValue.split(" ")[0];
-            localAccessMap.get(type)
-                    .getLocalEntriesWithMatchedRules()
-                    .merge(lastname,
-                           matchedRulesSet,
-                           (set1, set2) -> Stream.concat(set1.stream(), set2.stream())
-                                   .collect(Collectors.toSet()));
-        }
     }
 
 
@@ -186,10 +278,22 @@ public class Dictionary {
      * Recommends a text entity for inclusion in every dictionary model without separating the last name.
      *
      * @param textEntity The {@link TextEntity} to be recommended.
+     * @param level      The level of the dictionary where the recommendation should be added.
+     */
+    public void recommendEverywhere(TextEntity textEntity, Level level) {
+
+        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false, level);
+    }
+
+
+    /**
+     * Recommends a text entity for inclusion in every dictionary model without separating the last name at the default level.
+     *
+     * @param textEntity The {@link TextEntity} to be recommended.
      */
     public void recommendEverywhere(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
+        recommendEverywhere(textEntity, getDefaultLevel(textEntity.type()));
     }
 
 
@@ -197,10 +301,22 @@ public class Dictionary {
      * Recommends a text entity for inclusion in every dictionary model with the last name added separately.
      *
      * @param textEntity The {@link TextEntity} to be recommended.
+     * @param level      The level of the dictionary where the recommendation should be added.
+     */
+    public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity, Level level) {
+
+        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true, level);
+    }
+
+
+    /**
+     * Recommends a text entity for inclusion in every dictionary model with the last name added separately at the default level.
+     *
+     * @param textEntity The {@link TextEntity} to be recommended.
      */
     public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
+        recommendEverywhereWithLastNameSeparately(textEntity, getDefaultLevel(textEntity.type()));
     }
 
 
@@ -208,11 +324,22 @@ public class Dictionary {
      * Adds multiple author names contained within a text entity as recommendations in the dictionary.
      *
      * @param textEntity The {@link TextEntity} containing author names to be added.
+     * @param level      The level of the dictionary where the recommendations should be added.
+     */
+    public void addMultipleAuthorsAsRecommendation(TextEntity textEntity, Level level) {
+
+        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true, level));
+    }
+
+
+    /**
+     * Adds multiple author names contained within a text entity as recommendations in the dictionary at the default level.
+     *
+     * @param textEntity The {@link TextEntity} containing author names to be added.
      */
     public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
 
-        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true));
+        addMultipleAuthorsAsRecommendation(textEntity, getDefaultLevel(textEntity.type()));
-
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java

@@ -1,13 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
-import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 
 import lombok.RequiredArgsConstructor;
@@ -17,24 +19,12 @@ import lombok.SneakyThrows;
 @RequiredArgsConstructor
 public class DictionaryFactory {
 
-    private final RedactionServiceSettings settings;
-
 
     @SneakyThrows
     public Dictionary create(List<DictionaryModel> dictionaryModels, DictionaryVersion dictionaryVersion) {
 
-        // Generate the combined dictionary values map
         Map<DictionaryIdentifier, List<String>> combinedMap = generateCombinedDictionaryValuesMap(dictionaryModels);
-
+        DictionarySearch dictionarySearch = new DoubleArrayTrieDictionarySearch(combinedMap);
-        // Determine which DictionarySearch implementation to use
-        DictionarySearch dictionarySearch;
-        if (settings.isPriorityMode()) {
-            dictionarySearch = new HyperscanDictionarySearch(combinedMap);
-        } else {
-            dictionarySearch = new TrieDictionarySearch(combinedMap);
-        }
-
-        // Instantiate and return the Dictionary
         return new Dictionary(dictionaryModels, dictionaryVersion, dictionarySearch);
     }
 
@@ -42,62 +32,45 @@ public class DictionaryFactory {
     private Map<DictionaryIdentifier, List<String>> generateCombinedDictionaryValuesMap(List<DictionaryModel> dictionaryModels) {
 
         Map<DictionaryIdentifier, List<String>> combinedValuesMap = new HashMap<>();
-
         for (DictionaryModel model : dictionaryModels) {
-
+            addDictionaryEntries(combinedValuesMap, model);
-            DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive());
-            DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(),
-                                                                                     EntityType.DICTIONARY_REMOVAL,
-                                                                                     model.isDossierDictionary(),
-                                                                                     !model.isCaseInsensitive());
-            DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(),
-                                                                                    EntityType.FALSE_POSITIVE,
-                                                                                    model.isDossierDictionary(),
-                                                                                    !model.isCaseInsensitive());
-            DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(),
-                                                                                           EntityType.FALSE_RECOMMENDATION,
-                                                                                           model.isDossierDictionary(),
-                                                                                           !model.isCaseInsensitive());
-
-            List<String> entryValues = model.getEntries()
-                    .stream()
-                    .filter(e -> !e.isDeleted())
-                    .map(DictionaryEntry::getValue)
-                    .toList();
-
-            List<String> deletedEntryValues = model.getEntries()
-                    .stream()
-                    .filter(DictionaryEntry::isDeleted)
-                    .map(DictionaryEntry::getValue)
-                    .toList();
-
-            List<String> falsePositiveValues = model.getFalsePositives()
-                    .stream()
-                    .filter(e -> !e.isDeleted())
-                    .map(DictionaryEntry::getValue)
-                    .toList();
-
-            List<String> falseRecommendationValues = model.getFalseRecommendations()
-                    .stream()
-                    .filter(e -> !e.isDeleted())
-                    .map(DictionaryEntry::getValue)
-                    .toList();
-
-            if (!entryValues.isEmpty()) {
-                combinedValuesMap.put(entriesIdentifier, entryValues);
         }
-            if (!deletedEntryValues.isEmpty()) {
-                combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues);
-            }
-            if (!falsePositiveValues.isEmpty()) {
-                combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues);
-            }
-            if (!falseRecommendationValues.isEmpty()) {
-                combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues);
-            }
-        }
-
         return combinedValuesMap;
     }
 
+
+    private void addDictionaryEntries(Map<DictionaryIdentifier, List<String>> combinedValuesMap, DictionaryModel model) {
+
+        addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.ENTITY), filterValues(model.getEntries(), false));
+        addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.FALSE_POSITIVE), filterValues(model.getFalsePositives(), false));
+        addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.FALSE_RECOMMENDATION), filterValues(model.getFalseRecommendations(), false));
+
+        if (model.isDossierDictionary()) {
+            addValuesToMap(combinedValuesMap, createIdentifier(model, EntityType.DICTIONARY_REMOVAL), filterValues(model.getEntries(), true));
+        }
+    }
+
+
+    private DictionaryIdentifier createIdentifier(DictionaryModel model, EntityType entityType) {
+
+        return new DictionaryIdentifier(model.getType(), entityType, model.isDossierDictionary(), !model.isCaseInsensitive());
+    }
+
+
+    private List<String> filterValues(Set<DictionaryEntryModel> entries, boolean isDeleted) {
+
+        return entries.stream()
+                .filter(entry -> entry.isDeleted() == isDeleted)
+                .map(DictionaryEntry::getValue)
+                .toList();
+    }
+
+
+    private void addValuesToMap(Map<DictionaryIdentifier, List<String>> map, DictionaryIdentifier key, List<String> values) {
+
+        if (!values.isEmpty()) {
+            map.computeIfAbsent(key, k -> new ArrayList<>()).addAll(values);
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIdentifierTrie.java

@@ -5,10 +5,10 @@ import org.ahocorasick.trie.PayloadTrie;
 
 import java.util.Collection;
 
-public class DictionaryTrie {
+public class DictionaryIdentifierTrie {
     private final PayloadTrie<DictionaryIdentifier> trie;
 
-    private DictionaryTrie(PayloadTrie<DictionaryIdentifier> trie) {
+    private DictionaryIdentifierTrie(PayloadTrie<DictionaryIdentifier> trie) {
         this.trie = trie;
     }
 
@@ -36,8 +36,8 @@ public class DictionaryTrie {
             return this;
         }
 
-        public DictionaryTrie build() {
+        public DictionaryIdentifierTrie build() {
-            return new DictionaryTrie(builder.build());
+            return new DictionaryIdentifierTrie(builder.build());
         }
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java

@@ -10,14 +10,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
  */
 public interface DictionarySearch {
 
-    /**
-     * Checks if at least one dictionary entry matches the given text.
-     *
-     * @param text The text to search within.
-     * @return True if at least one match is found, otherwise false.
-     */
-    boolean atLeastOneMatches(String text);
-
     /**
      * Retrieves a list of match boundaries within the given text.
      *

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java

@@ -0,0 +1,126 @@
+package com.iqser.red.service.redaction.v1.server.model.dictionary;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+public class DoubleArrayTrieDictionarySearch implements DictionarySearch {
+
+    private final AhoCorasickDoubleArrayTrie<List<DictionaryIdentifierWithKeyword>> trie;
+
+
+    public DoubleArrayTrieDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
+
+        trie = new AhoCorasickDoubleArrayTrie<>();
+        trie.build(computeStringIdentifiersMap(dictionaryValues));
+    }
+
+
+    private static Map<String, List<DictionaryIdentifierWithKeyword>> computeStringIdentifiersMap(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
+
+        Map<String, List<DictionaryIdentifierWithKeyword>> stringToIdentifiersMap = new HashMap<>();
+        for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
+            DictionaryIdentifier identifier = entry.getKey();
+            List<String> values = entry.getValue();
+            for (String value : values) {
+                DictionaryIdentifierWithKeyword idWithKeyword = new DictionaryIdentifierWithKeyword(identifier, value);
+                stringToIdentifiersMap.computeIfAbsent(value.toLowerCase(Locale.ROOT), k -> new ArrayList<>()).add(idWithKeyword);
+            }
+        }
+        return stringToIdentifiersMap;
+    }
+
+
+    @Override
+    public Stream<MatchTextRange> getBoundaries(CharSequence text) {
+
+        String lowerText = text.toString().toLowerCase(Locale.ROOT);
+        return getMatchTextRangeStream(text, lowerText, 0);
+    }
+
+
+    @Override
+    public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
+
+        return getBoundaries(textBlock, textBlock.getTextRange());
+    }
+
+
+    @Override
+    public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
+
+        String subText = text.subSequence(region.start(), region.end()).toString();
+        String lowerSubText = subText.toLowerCase(Locale.ROOT);
+        return getMatchTextRangeStream(text, lowerSubText, region.start());
+    }
+
+
+    @Override
+    public Stream<MatchPosition> getMatches(String text) {
+
+        List<MatchPosition> matches = new ArrayList<>();
+        String lowerText = text.toLowerCase(Locale.ROOT);
+        List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(lowerText);
+
+        for (AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit : hits) {
+            String matchedText = text.substring(hit.begin, hit.end);
+            List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
+
+            for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
+                MatchPosition matchPosition = new MatchPosition(idkw.identifier, hit.begin, hit.end);
+                if (idkw.identifier.caseSensitive()) {
+                    if (matchedText.equals(idkw.keyword)) {
+                        matches.add(matchPosition);
+                    }
+                } else {
+                    matches.add(matchPosition);
+                }
+            }
+        }
+        return matches.stream();
+    }
+
+
+    private Stream<MatchTextRange> getMatchTextRangeStream(CharSequence text, String lowerText, int offset) {
+
+        List<MatchTextRange> matches = new ArrayList<>();
+        List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(lowerText);
+
+        for (AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit : hits) {
+            addMatchesForHit(text, matches, hit, offset);
+        }
+        return matches.stream();
+    }
+
+
+    private void addMatchesForHit(CharSequence text, List<MatchTextRange> matches, AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit, int offset) {
+
+        int start = hit.begin + offset;
+        int end = hit.end + offset;
+        String matchedText = text.subSequence(start, end).toString();
+        List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
+
+        for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
+            if (idkw.identifier.caseSensitive()) {
+                if (matchedText.equals(idkw.keyword)) {
+                    matches.add(new MatchTextRange(idkw.identifier, new TextRange(start, end)));
+                }
+            } else {
+                matches.add(new MatchTextRange(idkw.identifier, new TextRange(start, end)));
+            }
+        }
+    }
+
+
+    private record DictionaryIdentifierWithKeyword(DictionaryIdentifier identifier, String keyword) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java

@@ -1,172 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.dictionary;
-
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;
-
-import com.gliwka.hyperscan.wrapper.CompileErrorException;
-import com.gliwka.hyperscan.wrapper.Database;
-import com.gliwka.hyperscan.wrapper.Expression;
-import com.gliwka.hyperscan.wrapper.ExpressionFlag;
-import com.gliwka.hyperscan.wrapper.Match;
-import com.gliwka.hyperscan.wrapper.Scanner;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-
-public class HyperscanDictionarySearch implements DictionarySearch {
-
-    private final Map<Integer, DictionaryIdentifier> patternIdMap = new ConcurrentHashMap<>();
-    private AtomicInteger patternIdCounter = new AtomicInteger(0);
-    private Database database;
-
-
-    public HyperscanDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) throws CompileErrorException {
-
-        Map<DictionaryIdentifier, List<String>> entries = new HashMap<>(dictionaryValues);
-        this.database = compileDatabase(entries, this.patternIdMap);
-    }
-
-
-    private Database compileDatabase(Map<DictionaryIdentifier, List<String>> entries, Map<Integer, DictionaryIdentifier> patternIdMap) throws CompileErrorException {
-
-        if (entries.isEmpty()) {
-            return null;
-        }
-
-        LinkedList<Expression> expressions = new LinkedList<>();
-
-        for (Map.Entry<DictionaryIdentifier, List<String>> entry : entries.entrySet()) {
-            DictionaryIdentifier identifier = entry.getKey();
-            for (String value : entry.getValue()) {
-                int patternId = patternIdCounter.incrementAndGet();
-                patternIdMap.put(patternId, identifier);
-
-                EnumSet<ExpressionFlag> flags = EnumSet.noneOf(ExpressionFlag.class);
-                if (!identifier.caseSensitive()) {
-                    flags.add(ExpressionFlag.CASELESS);
-                }
-                // escape regex characters as we only use literals
-                expressions.add(new Expression(Pattern.quote(value), flags, patternId));
-            }
-        }
-
-        // Build the database
-        return Database.compile(expressions);
-    }
-
-
-    @Override
-    public boolean atLeastOneMatches(String text) {
-
-        if (this.database == null) {
-            return false;
-        }
-
-        try (Scanner scanner = new Scanner()) {
-            scanner.allocScratch(this.database);
-            List<Match> matches = scanner.scan(this.database, text);
-            return !matches.isEmpty();
-        }
-    }
-
-
-    @Override
-    public Stream<MatchTextRange> getBoundaries(CharSequence text) {
-
-        List<MatchTextRange> boundaries = new ArrayList<>();
-
-        if (this.database == null) {
-            return Stream.empty();
-        }
-
-        try (Scanner scanner = new Scanner()) {
-            scanner.allocScratch(this.database);
-            List<Match> matches = scanner.scan(this.database, text.toString());
-            for (Match match : matches) {
-                int start = (int) match.getStartPosition();
-                int end = (int) match.getEndPosition();
-                int patternId = match.getMatchedExpression().getId();
-                DictionaryIdentifier identifier = patternIdMap.get(patternId);
-                boundaries.add(new MatchTextRange(identifier, new TextRange(start, end)));
-            }
-        }
-
-        return boundaries.stream();
-    }
-
-
-    @Override
-    public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
-
-        return getBoundaries(textBlock, textBlock.getTextRange());
-    }
-
-
-    @Override
-    public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
-
-        List<MatchTextRange> boundaries = new ArrayList<>();
-
-        if (this.database == null) {
-            return Stream.empty();
-        }
-
-        CharSequence subText = text.subSequence(region.start(), region.end());
-
-        try (Scanner scanner = new Scanner()) {
-            scanner.allocScratch(this.database);
-            List<Match> matches = scanner.scan(this.database, subText.toString());
-            for (Match match : matches) {
-                int start = (int) (match.getStartPosition() + region.start());
-                int end = (int) (match.getEndPosition() + region.start());
-                int patternId = match.getMatchedExpression().getId();
-                DictionaryIdentifier identifier = patternIdMap.get(patternId);
-                boundaries.add(new MatchTextRange(identifier, new TextRange(start, end)));
-            }
-        }
-
-        return boundaries.stream();
-    }
-
-
-    @Override
-    public Stream<MatchPosition> getMatches(String text) {
-
-        List<MatchPosition> matchPositions = new ArrayList<>();
-
-        if (this.database == null) {
-            return Stream.empty();
-        }
-
-        try (Scanner scanner = new Scanner()) {
-            scanner.allocScratch(this.database);
-            List<Match> matches = scanner.scan(this.database, text);
-            for (Match match : matches) {
-                int start = (int) match.getStartPosition();
-                int end = (int) match.getEndPosition();
-                int patternId = match.getMatchedExpression().getId();
-                DictionaryIdentifier identifier = patternIdMap.get(patternId);
-                matchPositions.add(new MatchPosition(identifier, start, end));
-            }
-        }
-
-        return matchPositions.stream();
-    }
-
-
-    public void close() {
-
-        if (this.database != null) {
-            this.database.close();
-            this.database = null;
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java

@@ -4,7 +4,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
@@ -12,15 +11,14 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 
 public class TrieDictionarySearch implements DictionarySearch {
 
-    private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new ConcurrentHashMap<>();
+    private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new HashMap<>();
-    private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new ConcurrentHashMap<>();
+    private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new HashMap<>();
-    private final DictionaryTrie caseSensitiveTrie;
+    private final DictionaryIdentifierTrie caseSensitiveTrie;
-    private final DictionaryTrie caseInsensitiveTrie;
+    private final DictionaryIdentifierTrie caseInsensitiveTrie;
 
 
     public TrieDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
 
-        System.out.println("TEST ---> DictionarySearchImplementation CTOR");
         for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
             DictionaryIdentifier identifier = entry.getKey();
             List<String> values = entry.getValue();
@@ -36,12 +34,12 @@ public class TrieDictionarySearch implements DictionarySearch {
     }
 
 
-    private DictionaryTrie createTrie(Map<DictionaryIdentifier, List<String>> entries, boolean ignoreCase) {
+    private DictionaryIdentifierTrie createTrie(Map<DictionaryIdentifier, List<String>> entries, boolean ignoreCase) {
 
         if (entries.isEmpty()) {
             return null;
         }
-        DictionaryTrie.Builder builder = new DictionaryTrie.Builder();
+        DictionaryIdentifierTrie.Builder builder = new DictionaryIdentifierTrie.Builder();
         if (ignoreCase) {
             builder.ignoreCase();
         }
@@ -54,7 +52,6 @@ public class TrieDictionarySearch implements DictionarySearch {
     }
 
 
-    @Override
     public boolean atLeastOneMatches(String text) {
 
         if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) {
@@ -101,7 +98,7 @@ public class TrieDictionarySearch implements DictionarySearch {
     }
 
 
-    private void addMatchTextRangesForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchTextRange> matches, CharSequence text) {
+    private void addMatchTextRangesForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryIdentifierTrie trie, List<MatchTextRange> matches, CharSequence text) {
 
         if (!entries.isEmpty() && trie != null) {
             matches.addAll(trie.parseText(text)
@@ -116,7 +113,7 @@ public class TrieDictionarySearch implements DictionarySearch {
                                            TextRange region,
                                            List<MatchTextRange> matches,
                                            Map<DictionaryIdentifier, List<String>> entries,
-                                           DictionaryTrie trie) {
+                                           DictionaryIdentifierTrie trie) {
 
         if (!entries.isEmpty() && trie != null) {
             CharSequence subSequence = text.subSequence(region.start(), region.end());
@@ -128,7 +125,7 @@ public class TrieDictionarySearch implements DictionarySearch {
     }
 
 
-    private void addMatchPositionsForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchPosition> matches, String text) {
+    private void addMatchPositionsForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryIdentifierTrie trie, List<MatchPosition> matches, String text) {
 
         if (!entries.isEmpty() && trie != null) {
             matches.addAll(trie.parseText(text)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -31,7 +31,7 @@ public class DictionarySearchService {
     @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
     public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
 
-        semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node));
+        semanticNodes.parallelStream().forEach(node -> addDictionaryEntities(dictionary, node));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -223,7 +223,7 @@ public class DictionaryService {
                                    combinedEntries,
                                    combinedFalsePositives,
                                    combinedFalseRecommendations,
-                                   type.isDossierDictionaryOnly());
+                                   dossierId != null);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -58,21 +58,6 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     @Autowired
     DictionaryCacheService dictionaryCacheService;
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import(LayoutParsingServiceProcessorConfiguration.class)
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
 
     @BeforeEach
     public void stubClients() {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java

@@ -1,147 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.graph;
-
-import static org.springframework.test.util.AssertionErrors.assertEquals;
-
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.HyperscanDictionarySearch;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.gliwka.hyperscan.wrapper.CompileErrorException;
-
-@Disabled
-public class DictionaryPerformanceTest {
-
-    private static final int LARGE_DICTIONARY_SIZE = 5_000;
-    private static final int LARGE_TEXT_REPETITIONS = 50_000;
-    private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
-                                                    + "Entity_1 match text. Recommendation_1 also here. Random text continues. ";
-
-    @Test
-    public void testLargeFileWithLargeDictionaries() {
-
-        Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
-
-        // Generate dictionary values with specific terms for matching
-        IntStream.range(0, 6)
-                .forEach(i -> {
-                    EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION;
-                    boolean caseSensitive = i % 2 == 0;
-
-                    DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true,
-                                                                               caseSensitive);
-                    List<String> dictionary = generateLargeDictionary();
-
-                    // Add specific terms that are included in the large text for matches
-                    if (i == 0) {
-                        dictionary.add("Entity_1");
-                    }
-                    if (i == 1) {
-                        dictionary.add("recommendation_1");
-                    }
-                    dictionaryValues.put(identifier, dictionary);
-                });
-
-        // Measure construction time for DictionarySearchImplementation
-        long dictionaryTrieConstructionStart = System.currentTimeMillis();
-        TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(dictionaryValues);
-        long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart;
-
-        // Measure construction time for SearchImplementations
-        long searchTrieConstructionStart = System.currentTimeMillis();
-        List<SearchImplementation> searchImplementations = dictionaryValues.entrySet().stream()
-                .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive())).toList();
-        long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart;
-
-        // Measure construction time for HyperSearchImplementation
-        long hyperSearchConstructionStart = System.currentTimeMillis();
-        HyperscanDictionarySearch hyperSearchImpl = null;
-        try {
-            hyperSearchImpl = new HyperscanDictionarySearch(dictionaryValues);
-        } catch (CompileErrorException e) {
-            assert false : "Failed to compile HyperSearchImplementation: " + e.getMessage();
-        }
-        long hyperSearchConstructionDuration = System.currentTimeMillis() - hyperSearchConstructionStart;
-
-        String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
-
-        // Measure search time for DictionarySearchImplementation
-        long dictionarySearchStart = System.currentTimeMillis();
-        List<TrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl
-                .getBoundariesAsList(largeText);
-        long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart;
-
-        // Measure search time for SearchImplementations
-        long searchImplStart = System.currentTimeMillis();
-        List<TextRange> searchMatches = new ArrayList<>();
-        for (SearchImplementation searchImpl : searchImplementations) {
-            searchMatches.addAll(searchImpl.getBoundaries(largeText));
-        }
-        long searchImplDuration = System.currentTimeMillis() - searchImplStart;
-
-        // Measure search time for HyperSearchImplementation
-        long hyperSearchStart = System.currentTimeMillis();
-        List<DictionarySearch.MatchPosition> hyperMatches = hyperSearchImpl.getMatchesAsList(largeText);
-        long hyperSearchDuration = System.currentTimeMillis() - hyperSearchStart;
-
-        System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration);
-        System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration,
-                          dictionaryMatches.size());
-        System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration);
-        System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration,
-                          searchMatches.size());
-        System.out.printf("HyperSearchImplementation construction took %d ms\n", hyperSearchConstructionDuration);
-        System.out.printf("HyperSearchImplementation search took %d ms and found %d matches\n", hyperSearchDuration,
-                          hyperMatches.size());
-
-        assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() && !hyperMatches.isEmpty()
-                : "All implementations should find entities.";
-
-        // Ensure all implementations found the same number of matches
-        assertEquals("DictionarySearchImplementation and SearchImplementations should find the same number of matches",
-                     dictionaryMatches.size(), searchMatches.size());
-        assertEquals("DictionarySearchImplementation and HyperSearchImplementation should find the same number of matches",
-                     dictionaryMatches.size(), hyperMatches.size());
-
-        // Close the HyperSearchImplementation to release resources
-        hyperSearchImpl.close();
-    }
-
-    private List<String> generateLargeDictionary() {
-        return IntStream.range(0, LARGE_DICTIONARY_SIZE)
-                .mapToObj(i -> RandomStringGenerator.generateRandomString())
-                .collect(Collectors.toList());
-    }
-
-    static final class RandomStringGenerator {
-
-        private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
-        private static final int STRING_LENGTH = 50;
-        private static final SecureRandom RANDOM = new SecureRandom();
-
-        public static String generateRandomString() {
-
-            StringBuilder sb = new StringBuilder(STRING_LENGTH);
-            for (int i = 0; i < STRING_LENGTH; i++) {
-                int index = RANDOM.nextInt(CHARACTERS.length());
-                sb.append(CHARACTERS.charAt(index));
-            }
-            return sb.toString();
-        }
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java

@@ -0,0 +1,176 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.jupiter.api.Test;
+
+import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DoubleArrayTrieDictionarySearch;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+
+public class DictionarySearchImplementationsTest {
+
+    private static final int LARGE_DICTIONARY_SIZE = 4_000;
+    private static final int LARGE_TEXT_REPETITIONS = 500_000;
+    private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
+                                                    + "Entity_1 match text. Recommendation_1 also here. Random text continues. ";
+
+
+    @Test
+//@Disabled
+    public void performanceTest() {
+
+        Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
+
+        // Generate dictionary values with specific terms for matching
+        IntStream.range(0, 6)
+                .forEach(i -> {
+                    EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION;
+                    boolean caseSensitive = i % 2 == 0;
+
+                    DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive);
+                    List<String> dictionary = generateLargeDictionary();
+
+                    // Add specific terms that are included in the large text for matches
+                    if (i == 0) {
+                        dictionary.add("Entity_1");
+                    }
+                    if (i == 1) {
+                        dictionary.add("recommendation_1");
+                    }
+                    dictionaryValues.put(identifier, dictionary);
+                });
+
+        // Measure construction time for TrieDictionarySearch
+        long trieDictionaryConstructionStart = System.currentTimeMillis();
+        TrieDictionarySearch trieDictionarySearchImpl = new TrieDictionarySearch(dictionaryValues);
+        long trieDictionaryConstructionDuration = System.currentTimeMillis() - trieDictionaryConstructionStart;
+
+        // Measure construction time for SearchImplementations
+        long searchTrieConstructionStart = System.currentTimeMillis();
+        List<SearchImplementation> searchImplementations = dictionaryValues.entrySet()
+                .stream()
+                .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive()))
+                .toList();
+        long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart;
+
+        // Measure construction time for DoubleArrayTrieDictionarySearch
+        long doubleArrayTrieConstructionStart = System.currentTimeMillis();
+        DoubleArrayTrieDictionarySearch doubleArrayTrieSearchImpl = new DoubleArrayTrieDictionarySearch(dictionaryValues);
+        long doubleArrayTrieConstructionDuration = System.currentTimeMillis() - doubleArrayTrieConstructionStart;
+
+        String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
+
+        // Measure search time for TrieDictionarySearch
+        long trieDictionarySearchStart = System.currentTimeMillis();
+        List<TrieDictionarySearch.MatchTextRange> trieDictionaryMatches = trieDictionarySearchImpl.getBoundariesAsList(largeText);
+        long trieDictionarySearchDuration = System.currentTimeMillis() - trieDictionarySearchStart;
+
+        // Measure search time for SearchImplementations
+        long searchImplStart = System.currentTimeMillis();
+        List<TextRange> searchMatches = new ArrayList<>();
+        for (SearchImplementation searchImpl : searchImplementations) {
+            searchMatches.addAll(searchImpl.getBoundaries(largeText));
+        }
+        long searchImplDuration = System.currentTimeMillis() - searchImplStart;
+
+        // Measure search time for DoubleArrayTrieDictionarySearch
+        long doubleArrayTrieSearchStart = System.currentTimeMillis();
+        List<DictionarySearch.MatchTextRange> doubleArrayTrieMatches = doubleArrayTrieSearchImpl.getBoundariesAsList(largeText);
+        long doubleArrayTrieSearchDuration = System.currentTimeMillis() - doubleArrayTrieSearchStart;
+
+        // Output the performance results
+        System.out.printf("TrieDictionarySearch construction took %d ms\n", trieDictionaryConstructionDuration);
+        System.out.printf("TrieDictionarySearch search took %d ms and found %d matches\n", trieDictionarySearchDuration, trieDictionaryMatches.size());
+        System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration);
+        System.out.printf("Combined SearchImplementation search took %d ms and found %d matches\n", searchImplDuration, searchMatches.size());
+        System.out.printf("DoubleArrayTrieDictionarySearch construction took %d ms\n", doubleArrayTrieConstructionDuration);
+        System.out.printf("DoubleArrayTrieDictionarySearch search took %d ms and found %d matches\n", doubleArrayTrieSearchDuration, doubleArrayTrieMatches.size());
+
+        // Assert that all implementations found matches
+        assert !trieDictionaryMatches.isEmpty()
+               && !searchMatches.isEmpty()
+               && !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities.";
+
+        // Ensure all implementations found the same number of matches
+        assertEquals(trieDictionaryMatches.size(), searchMatches.size(), "Mismatch between TrieDictionarySearch and SearchImplementations");
+        assertEquals(trieDictionaryMatches.size(), doubleArrayTrieMatches.size(), "Mismatch between TrieDictionarySearch and DoubleArrayTrieDictionarySearch");
+    }
+
+
+    private List<String> generateLargeDictionary() {
+
+        return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString())
+                .collect(Collectors.toList());
+    }
+
+
+    static final class RandomStringGenerator {
+
+        private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+        private static final int STRING_LENGTH = 50;
+        private static final SecureRandom RANDOM = new SecureRandom();
+
+
+        public static String generateRandomString() {
+
+            StringBuilder sb = new StringBuilder(STRING_LENGTH);
+            for (int i = 0; i < STRING_LENGTH; i++) {
+                int index = RANDOM.nextInt(CHARACTERS.length());
+                sb.append(CHARACTERS.charAt(index));
+            }
+            return sb.toString();
+        }
+
+    }
+
+
+    @Test
+    public void testMultiplePayloads() {
+
+        TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(Map.of(new DictionaryIdentifier("type1", EntityType.ENTITY, false, false),
+                                                                                    List.of("apple", "banana"),
+                                                                                    new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false),
+                                                                                    List.of("apple", "orange"),
+                                                                                    new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false),
+                                                                                    List.of("apple", "kiwi")));
+
+        List<TrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundariesAsList(
+                "an apple is delicious, a banana and a kiwi as well. orange is a color.");
+
+        assertEquals(dictionaryMatches.size(), 6);
+
+    }
+
+
+    @Test
+    public void testDoubleArrayTrie() {
+
+        Map<String, List<String>> map = new HashMap<>();
+        String[] keyArray = new String[]{"hers", "his", "she", "he"};
+        for (String key : keyArray) {
+            map.put(key, List.of(key, key, key));
+        }
+
+        AhoCorasickDoubleArrayTrie<List<String>> acdat = new AhoCorasickDoubleArrayTrie<>();
+        acdat.build(map);
+
+        final String text = "uhers";
+        List<AhoCorasickDoubleArrayTrie.Hit<List<String>>> wordList = acdat.parseText(text);
+        assertEquals(wordList.size(), 2);
+    }
+
+}