diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 8896590a..81d64077 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -87,6 +87,7 @@ dependencies { implementation("org.reflections:reflections:0.10.2") implementation("com.opencsv:opencsv:5.9") + implementation("com.gliwka.hyperscan:hyperscan:5.4.11-3.0.0") implementation("com.joestelmach:natty:0.13") testImplementation(project(":rules-management")) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index ebdf6779..39c9abae 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -28,6 +28,8 @@ public class RedactionServiceSettings { private boolean priorityMode; + private long firstLevelDictionaryCacheMaximumSize = 1000; + private long dictionaryCacheMaximumSize = 100; private int dictionaryCacheExpireAfterAccessDays = 3; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index 7a180994..e18c13e0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -24,33 +24,26 @@ import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundExcepti import lombok.Data; import lombok.Getter; -import lombok.NoArgsConstructor; /** * A class representing a dictionary used for redaction processes, containing various dictionary models and their versions. */ @Data -@NoArgsConstructor public class Dictionary { - @Getter - private List dictionaryModels; - // todo: dossier and dossier template level DictionaryModels override each other - // at the moment there are no problems because they always have the same rank / hint information - // but it should be changed so that the localAccessMap contains all models - private Map localAccessMap = new HashMap<>(); + private final Map localAccessMap = new HashMap<>(); @Getter - private DictionaryVersion version; + private final DictionaryVersion version; - private DictionarySearchImplementation dictionarySearch; + private final DictionarySearch dictionarySearch; - public Dictionary(List dictionaryModels, DictionaryVersion version) { + Dictionary(List dictionaryModels, DictionaryVersion version, DictionarySearch dictionarySearch) { - this.dictionaryModels = dictionaryModels; - this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm)); + dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm)); this.version = version; + this.dictionarySearch = dictionarySearch; } @@ -70,11 +63,19 @@ public class Dictionary { */ public boolean hasLocalEntries() { - return dictionaryModels.stream() + return getDictionaryModels().stream() .anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty()); } + public List getDictionaryModels() { + + return localAccessMap.values() + .stream() + .toList(); + } + + public Set getTypes() { return localAccessMap.keySet(); @@ -130,77 +131,6 @@ public class Dictionary { } - public DictionarySearchImplementation getDictionarySearch() { - - if (dictionarySearch == null) { - dictionarySearch = new DictionarySearchImplementation(generateCombinedDictionaryValuesMap()); - } - return dictionarySearch; - } - - - private Map> generateCombinedDictionaryValuesMap() { - - Map> combinedValuesMap = new HashMap<>(); - - for (DictionaryModel model : dictionaryModels) { - - DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive()); - DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(), - EntityType.DICTIONARY_REMOVAL, - model.isDossierDictionary(), - !model.isCaseInsensitive()); - DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(), - EntityType.FALSE_POSITIVE, - model.isDossierDictionary(), - !model.isCaseInsensitive()); - DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(), - EntityType.FALSE_RECOMMENDATION, - model.isDossierDictionary(), - !model.isCaseInsensitive()); - - List entryValues = model.getEntries() - .stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .toList(); - - List deletedEntryValues = model.getEntries() - .stream() - .filter(DictionaryEntry::isDeleted) - .map(DictionaryEntry::getValue) - .toList(); - - List falsePositiveValues = model.getFalsePositives() - .stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .toList(); - - List falseRecommendationValues = model.getFalseRecommendations() - .stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .toList(); - - if (!entryValues.isEmpty()) { - combinedValuesMap.put(entriesIdentifier, entryValues); - } - if (!deletedEntryValues.isEmpty()) { - combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues); - } - if (!falsePositiveValues.isEmpty()) { - combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues); - } - if (!falseRecommendationValues.isEmpty()) { - combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues); - } - } - - return combinedValuesMap; - } - - /** * Adds a local dictionary entry of a specific type. * diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java new file mode 100644 index 00000000..16efc3bb --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java @@ -0,0 +1,103 @@ +package com.iqser.red.service.redaction.v1.server.model.dictionary; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +@Service +@RequiredArgsConstructor +public class DictionaryFactory { + + private final RedactionServiceSettings settings; + + + @SneakyThrows + public Dictionary create(List dictionaryModels, DictionaryVersion dictionaryVersion) { + + // Generate the combined dictionary values map + Map> combinedMap = generateCombinedDictionaryValuesMap(dictionaryModels); + + // Determine which DictionarySearch implementation to use + DictionarySearch dictionarySearch; + if (settings.isPriorityMode()) { + dictionarySearch = new HyperscanDictionarySearch(combinedMap); + } else { + dictionarySearch = new TrieDictionarySearch(combinedMap); + } + + // Instantiate and return the Dictionary + return new Dictionary(dictionaryModels, dictionaryVersion, dictionarySearch); + } + + + private Map> generateCombinedDictionaryValuesMap(List dictionaryModels) { + + Map> combinedValuesMap = new HashMap<>(); + + for (DictionaryModel model : dictionaryModels) { + + DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive()); + DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(), + EntityType.DICTIONARY_REMOVAL, + model.isDossierDictionary(), + !model.isCaseInsensitive()); + DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(), + EntityType.FALSE_POSITIVE, + model.isDossierDictionary(), + !model.isCaseInsensitive()); + DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(), + EntityType.FALSE_RECOMMENDATION, + model.isDossierDictionary(), + !model.isCaseInsensitive()); + + List entryValues = model.getEntries() + .stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .toList(); + + List deletedEntryValues = model.getEntries() + .stream() + .filter(DictionaryEntry::isDeleted) + .map(DictionaryEntry::getValue) + .toList(); + + List falsePositiveValues = model.getFalsePositives() + .stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .toList(); + + List falseRecommendationValues = model.getFalseRecommendations() + .stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .toList(); + + if (!entryValues.isEmpty()) { + combinedValuesMap.put(entriesIdentifier, entryValues); + } + if (!deletedEntryValues.isEmpty()) { + combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues); + } + if (!falsePositiveValues.isEmpty()) { + combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues); + } + if (!falseRecommendationValues.isEmpty()) { + combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues); + } + } + + return combinedValuesMap; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java new file mode 100644 index 00000000..e96ef54e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java @@ -0,0 +1,94 @@ +package com.iqser.red.service.redaction.v1.server.model.dictionary; + +import java.util.List; +import java.util.stream.Stream; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; + +/** + * Common interface for dictionary search implementations. + */ +public interface DictionarySearch { + + /** + * Checks if at least one dictionary entry matches the given text. + * + * @param text The text to search within. + * @return True if at least one match is found, otherwise false. + */ + boolean atLeastOneMatches(String text); + + /** + * Retrieves a list of match boundaries within the given text. + * + * @param text The text to search within. + * @return A list of MatchTextRange representing the boundaries of matches. + */ + default List getBoundariesAsList(CharSequence text) { + return getBoundaries(text).toList(); + } + + /** + * Retrieves a stream of match boundaries within the given text. + * + * @param text The text to search within. + * @return A stream of MatchTextRange representing the boundaries of matches. + */ + Stream getBoundaries(CharSequence text); + + /** + * Retrieves a list of match boundaries within a specified region of the text. + * + * @param text The text to search within. + * @param region The specific region of the text to search. + * @return A list of MatchTextRange representing the boundaries of matches. + */ + default List getBoundariesAsList(CharSequence text, TextRange region) { + return getBoundaries(text, region).toList(); + } + + /** + * Retrieves a stream of match boundaries within a specified region of the text. + * + * @param text The text to search within. + * @param region The specific region of the text to search. + * @return A stream of MatchTextRange representing the boundaries of matches. + */ + Stream getBoundaries(CharSequence text, TextRange region); + + /** + * Retrieves a stream of match boundaries within the given TextBlock. + * + * @param textBlock The TextBlock to search within. + * @return A stream of MatchTextRange representing the boundaries of matches. + */ + Stream getBoundaries(TextBlock textBlock); + + /** + * Retrieves a list of match positions within the given text. + * + * @param text The text to search within. + * @return A list of MatchPosition representing the positions of matches. + */ + default List getMatchesAsList(String text) { + return getMatches(text).toList(); + } + + /** + * Retrieves a stream of match positions within the given text. + * + * @param text The text to search within. + * @return A stream of MatchPosition representing the positions of matches. + */ + Stream getMatches(String text); + + /** + * Record representing the range of matched text along with its identifier. + */ + record MatchTextRange(DictionaryIdentifier identifier, TextRange textRange) {} + + /** + * Record representing the start and end positions of a match along with its identifier. + */ + record MatchPosition(DictionaryIdentifier identifier, int startIndex, int endIndex) {} +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java new file mode 100644 index 00000000..7c692d7d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java @@ -0,0 +1,172 @@ +package com.iqser.red.service.redaction.v1.server.model.dictionary; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import com.gliwka.hyperscan.wrapper.CompileErrorException; +import com.gliwka.hyperscan.wrapper.Database; +import com.gliwka.hyperscan.wrapper.Expression; +import com.gliwka.hyperscan.wrapper.ExpressionFlag; +import com.gliwka.hyperscan.wrapper.Match; +import com.gliwka.hyperscan.wrapper.Scanner; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; + +public class HyperscanDictionarySearch implements DictionarySearch { + + private final Map patternIdMap = new ConcurrentHashMap<>(); + private AtomicInteger patternIdCounter = new AtomicInteger(0); + private Database database; + + + public HyperscanDictionarySearch(Map> dictionaryValues) throws CompileErrorException { + + Map> entries = new HashMap<>(dictionaryValues); + this.database = compileDatabase(entries, this.patternIdMap); + } + + + private Database compileDatabase(Map> entries, Map patternIdMap) throws CompileErrorException { + + if (entries.isEmpty()) { + return null; + } + + LinkedList expressions = new LinkedList<>(); + + for (Map.Entry> entry : entries.entrySet()) { + DictionaryIdentifier identifier = entry.getKey(); + for (String value : entry.getValue()) { + int patternId = patternIdCounter.incrementAndGet(); + patternIdMap.put(patternId, identifier); + + EnumSet flags = EnumSet.noneOf(ExpressionFlag.class); + if (!identifier.caseSensitive()) { + flags.add(ExpressionFlag.CASELESS); + } + // escape regex characters as we only use literals + expressions.add(new Expression(Pattern.quote(value), flags, patternId)); + } + } + + // Build the database + return Database.compile(expressions); + } + + + @Override + public boolean atLeastOneMatches(String text) { + + if (this.database == null) { + return false; + } + + try (Scanner scanner = new Scanner()) { + scanner.allocScratch(this.database); + List matches = scanner.scan(this.database, text); + return !matches.isEmpty(); + } + } + + + @Override + public Stream getBoundaries(CharSequence text) { + + List boundaries = new ArrayList<>(); + + if (this.database == null) { + return Stream.empty(); + } + + try (Scanner scanner = new Scanner()) { + scanner.allocScratch(this.database); + List matches = scanner.scan(this.database, text.toString()); + for (Match match : matches) { + int start = (int) match.getStartPosition(); + int end = (int) match.getEndPosition(); + int patternId = match.getMatchedExpression().getId(); + DictionaryIdentifier identifier = patternIdMap.get(patternId); + boundaries.add(new MatchTextRange(identifier, new TextRange(start, end))); + } + } + + return boundaries.stream(); + } + + + @Override + public Stream getBoundaries(TextBlock textBlock) { + + return getBoundaries(textBlock, textBlock.getTextRange()); + } + + + @Override + public Stream getBoundaries(CharSequence text, TextRange region) { + + List boundaries = new ArrayList<>(); + + if (this.database == null) { + return Stream.empty(); + } + + CharSequence subText = text.subSequence(region.start(), region.end()); + + try (Scanner scanner = new Scanner()) { + scanner.allocScratch(this.database); + List matches = scanner.scan(this.database, subText.toString()); + for (Match match : matches) { + int start = (int) (match.getStartPosition() + region.start()); + int end = (int) (match.getEndPosition() + region.start()); + int patternId = match.getMatchedExpression().getId(); + DictionaryIdentifier identifier = patternIdMap.get(patternId); + boundaries.add(new MatchTextRange(identifier, new TextRange(start, end))); + } + } + + return boundaries.stream(); + } + + + @Override + public Stream getMatches(String text) { + + List matchPositions = new ArrayList<>(); + + if (this.database == null) { + return Stream.empty(); + } + + try (Scanner scanner = new Scanner()) { + scanner.allocScratch(this.database); + List matches = scanner.scan(this.database, text); + for (Match match : matches) { + int start = (int) match.getStartPosition(); + int end = (int) match.getEndPosition(); + int patternId = match.getMatchedExpression().getId(); + DictionaryIdentifier identifier = patternIdMap.get(patternId); + matchPositions.add(new MatchPosition(identifier, start, end)); + } + } + + return matchPositions.stream(); + } + + + public void close() { + + if (this.database != null) { + this.database.close(); + this.database = null; + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java similarity index 89% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java index 8cb8f003..39c56e53 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java @@ -4,20 +4,21 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; -public class DictionarySearchImplementation { +public class TrieDictionarySearch implements DictionarySearch { - private final Map> caseSensitiveEntries = new HashMap<>(); - private final Map> caseInsensitiveEntries = new HashMap<>(); + private final Map> caseSensitiveEntries = new ConcurrentHashMap<>(); + private final Map> caseInsensitiveEntries = new ConcurrentHashMap<>(); private final DictionaryTrie caseSensitiveTrie; private final DictionaryTrie caseInsensitiveTrie; - public DictionarySearchImplementation(Map> dictionaryValues) { + public TrieDictionarySearch(Map> dictionaryValues) { System.out.println("TEST ---> DictionarySearchImplementation CTOR"); for (Map.Entry> entry : dictionaryValues.entrySet()) { @@ -53,6 +54,7 @@ public class DictionarySearchImplementation { } + @Override public boolean atLeastOneMatches(String text) { if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) { @@ -62,12 +64,40 @@ public class DictionarySearchImplementation { } - public List getBoundaries(CharSequence text) { + @Override + public Stream getBoundaries(CharSequence text) { List matches = new ArrayList<>(); addMatchTextRangesForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text); addMatchTextRangesForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text); - return matches; + return matches.stream(); + } + + + @Override + public Stream getBoundaries(TextBlock textBlock) { + + return getBoundaries(textBlock, textBlock.getTextRange()); + } + + + @Override + public Stream getBoundaries(CharSequence text, TextRange region) { + + List matches = new ArrayList<>(); + addMatchTextRangesForTrie(text, region, matches, caseSensitiveEntries, caseSensitiveTrie); + addMatchTextRangesForTrie(text, region, matches, caseInsensitiveEntries, caseInsensitiveTrie); + return matches.stream(); + } + + + @Override + public Stream getMatches(String text) { + + List matches = new ArrayList<>(); + addMatchPositionsForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text); + addMatchPositionsForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text); + return matches.stream(); } @@ -82,21 +112,6 @@ public class DictionarySearchImplementation { } - public Stream getBoundaries(TextBlock textBlock) { - - return getBoundaries(textBlock, textBlock.getTextRange()); - } - - - public Stream getBoundaries(CharSequence text, TextRange region) { - - List matches = new ArrayList<>(); - addMatchTextRangesForTrie(text, region, matches, caseSensitiveEntries, caseSensitiveTrie); - addMatchTextRangesForTrie(text, region, matches, caseInsensitiveEntries, caseInsensitiveTrie); - return matches.stream(); - } - - private void addMatchTextRangesForTrie(CharSequence text, TextRange region, List matches, @@ -113,15 +128,6 @@ public class DictionarySearchImplementation { } - public List getMatches(String text) { - - List matches = new ArrayList<>(); - addMatchPositionsForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text); - addMatchPositionsForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text); - return matches; - } - - private void addMatchPositionsForTrie(Map> entries, DictionaryTrie trie, List matches, String text) { if (!entries.isEmpty() && trie != null) { @@ -132,13 +138,4 @@ public class DictionarySearchImplementation { } } - - public record MatchTextRange(DictionaryIdentifier identifier, TextRange textRange) { - - } - - public record MatchPosition(DictionaryIdentifier identifier, int startIndex, int endIndex) { - - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java index fa736613..f49f4056 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -102,7 +101,7 @@ public class AnalysisPreparationService { CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join(); - Dictionary dictionary = getDictionary(analyzeRequest); + Dictionary dictionary = dictionaryService.getDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Document document = documentFuture.get(); ImportedRedactions importedRedactions = importedRedactionsFuture.get(); @@ -193,7 +192,7 @@ public class AnalysisPreparationService { taskExecutor); CompletableFuture dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> { - Dictionary dictionary = getDictionary(analyzeRequest); + Dictionary dictionary = dictionaryService.getDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, reanalysisSetupData.document(), reanalysisInitialProcessingData.importedRedactions()); return new DictionaryAndNotFoundEntries(dictionary, notFoundEntries.notFoundManualRedactionEntries(), notFoundEntries.notFoundImportedEntries()); }, taskExecutor); @@ -251,15 +250,6 @@ public class AnalysisPreparationService { } - private Dictionary getDictionary(AnalyzeRequest analyzeRequest) { - - dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - return dictionary; - } - - private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions) { var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryCacheService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryCacheService.java new file mode 100644 index 00000000..8b10ca09 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryCacheService.java @@ -0,0 +1,111 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import org.springframework.stereotype.Service; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryRepresentation; +import com.iqser.red.service.redaction.v1.server.model.dictionary.TenantDictionary; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DictionaryCacheService { + + private final RedactionServiceSettings settings; + + private LoadingCache tenantDictionaryCache; + private Cache dictionaryCache; + + + @PostConstruct + protected void createCache() { + + tenantDictionaryCache = CacheBuilder.newBuilder() + .maximumSize(settings.getDictionaryCacheMaximumSize()) + .expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS) + .build(new CacheLoader<>() { + public TenantDictionary load(String key) { + + return new TenantDictionary(); + } + }); + + dictionaryCache = CacheBuilder.newBuilder() + .maximumSize(settings.getFirstLevelDictionaryCacheMaximumSize()) + .expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS) + .build(); + } + + + public void clearAllCaches() { + + tenantDictionaryCache.invalidateAll(); + dictionaryCache.invalidateAll(); + } + + + @SneakyThrows + public DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) { + + return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate() + .get(dossierTemplateId); + } + + + @SneakyThrows + public DictionaryRepresentation getDossierDictionary(String dossierId) { + + return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier() + .get(dossierId); + } + + + @SneakyThrows + public void addDictionaryRepresentationForDossierTemplate(String dossierTemplateId, DictionaryRepresentation dictionaryRepresentation) { + + tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation); + } + + + @SneakyThrows + public void addDictionaryRepresentationForDossier(String dossierId, DictionaryRepresentation dictionaryRepresentation) { + + tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation); + } + + + @SneakyThrows + public Optional getDictionary(String tenantId, String dossierId) { + + return Optional.ofNullable(dictionaryCache.getIfPresent(new DictionaryCacheKey(tenantId, dossierId))); + + } + + + @SneakyThrows + public void putDictionary(String tenantId, String dossierId, Dictionary newDictionary) { + + dictionaryCache.put(new DictionaryCacheKey(tenantId, dossierId), newDictionary); + + } + + + public record DictionaryCacheKey(String tenantId, String dossierId) { + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 4bb6fd16..fb643f4f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -7,9 +7,6 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index 05c323e4..91641adb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -8,37 +8,28 @@ import java.util.List; import java.util.Locale; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryEntries; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryFactory; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryRepresentation; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.model.dictionary.TenantDictionary; import com.knecon.fforesight.tenantcommons.TenantContext; import feign.FeignException; import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; -import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -50,32 +41,39 @@ public class DictionaryService { public static final String DEFAULT_COLOR = "#cccccc"; private final DictionaryClient dictionaryClient; - - private final RedactionServiceSettings settings; - - private LoadingCache tenantDictionaryCache; - private ConcurrentMap dictionaryCache = new ConcurrentHashMap<>(); + private final DictionaryCacheService dictionaryCacheService; + private final DictionaryFactory dictionaryFactory; - @PostConstruct - protected void createCaches() { + @SneakyThrows + @Observed(name = "DictionaryService", contextualName = "get-dictionary") + @Timed("redactmanager_getDictionary") + public Dictionary getDictionary(String dossierTemplateId, String dossierId) { - tenantDictionaryCache = CacheBuilder.newBuilder() - .maximumSize(settings.getDictionaryCacheMaximumSize()) - .expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS) - .build(new CacheLoader<>() { - public TenantDictionary load(String key) { + String tenantId = TenantContext.getTenantId(); - return new TenantDictionary(); - } - }); + Optional cachedDictionary = dictionaryCacheService.getDictionary(tenantId, dossierId); - } + if (cachedDictionary.isPresent()) { + log.debug("Dictionary found in cache"); + boolean isUpToDate = checkIfDictionaryIsUpToDate(dossierTemplateId, dossierId, cachedDictionary.get()); + if (isUpToDate) { + log.info("Returning cached Dictionary for tenantId: {}, dossierId: {}", tenantId, dossierId); + return cachedDictionary.get(); + } else { + log.info("Cached Dictionary is outdated for tenantId: {}, dossierId: {}", tenantId, dossierId); + } + } else { + log.info("No cached Dictionary found for tenantId: {}, dossierId: {}", tenantId, dossierId); + } + DictionaryVersion latestVersion = updateDictionary(dossierTemplateId, dossierId); + Dictionary newDictionary = buildDictionary(dossierTemplateId, dossierId, latestVersion); - public void clearTenantDictionaryCache() { + dictionaryCacheService.putDictionary(tenantId, dossierId, newDictionary); + log.info("Cached new Dictionary for tenantId: {}, dossierId: {}", tenantId, dossierId); - tenantDictionaryCache.invalidateAll(); + return newDictionary; } @@ -85,19 +83,24 @@ public class DictionaryService { public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) { log.debug("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId); - long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId); - var dossierTemplateDictionary = getDossierTemplateDictionary(dossierTemplateId); - if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) { - updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null); + + // Update template dictionary + long latestTemplateVersion = dictionaryClient.getVersion(dossierTemplateId); + DictionaryRepresentation templateDictRep = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId); + + if (templateDictRep == null || latestTemplateVersion > templateDictRep.getDictionaryVersion()) { + updateDictionaryEntry(dossierTemplateId, latestTemplateVersion, templateDictRep != null ? templateDictRep.getDictionaryVersion() : null, null); } - long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId); - var dossierDictionary = getDossierDictionary(dossierId); - if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) { - updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId); + // Update dossier dictionary + long latestDossierVersion = dictionaryClient.getVersionForDossier(dossierId); + DictionaryRepresentation dossierDictRep = dictionaryCacheService.getDossierDictionary(dossierId); + + if (dossierDictRep == null || latestDossierVersion > dossierDictRep.getDictionaryVersion()) { + updateDictionaryEntry(dossierTemplateId, latestDossierVersion, dossierDictRep != null ? dossierDictRep.getDictionaryVersion() : null, dossierId); } - return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build(); + return DictionaryVersion.builder().dossierTemplateVersion(latestTemplateVersion).dossierVersion(latestDossierVersion).build(); } @@ -106,57 +109,46 @@ public class DictionaryService { @Timed("redactmanager_getDictionaryIncrements") public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) { - DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId); + DictionaryVersion latestVersion = updateDictionary(dossierTemplateId, dossierId); Set newValues = new HashSet<>(); - List dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary(); + List templateDictionaries = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId).getDictionary(); - dictionaryModels.forEach(dictionaryModel -> { - dictionaryModel.getEntries() - .forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); - dictionaryModel.getFalsePositives() - .forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); - dictionaryModel.getFalseRecommendations() - .forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); + templateDictionaries.forEach(dictionaryModel -> { + addNewEntries(dictionaryModel, fromVersion.getDossierTemplateVersion(), newValues); }); if (dossierDictionaryExists(dossierId)) { - dictionaryModels = getDossierDictionary(dossierId).getDictionary(); - dictionaryModels.forEach(dictionaryModel -> { - dictionaryModel.getEntries() - .forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); - dictionaryModel.getFalsePositives() - .forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); - dictionaryModel.getFalseRecommendations() - .forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); + List dossierDictionaries = dictionaryCacheService.getDossierDictionary(dossierId).getDictionary(); + dossierDictionaries.forEach(dictionaryModel -> { + addNewEntries(dictionaryModel, fromVersion.getDossierVersion(), newValues); }); } - return new DictionaryIncrement(newValues, version); + return new DictionaryIncrement(newValues, latestVersion); + } + + + private void addNewEntries(DictionaryModel dictionaryModel, long versionThreshold, Set newValues) { + + dictionaryModel.getEntries() + .forEach(entry -> { + if (entry.getVersion() > versionThreshold) { + newValues.add(new DictionaryIncrementValue(entry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); + dictionaryModel.getFalsePositives() + .forEach(entry -> { + if (entry.getVersion() > versionThreshold) { + newValues.add(new DictionaryIncrementValue(entry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); + dictionaryModel.getFalseRecommendations() + .forEach(entry -> { + if (entry.getVersion() > versionThreshold) { + newValues.add(new DictionaryIncrementValue(entry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); } @@ -166,125 +158,18 @@ public class DictionaryService { try { DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation(); - var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, currentVersion, true) : dictionaryClient.getAllTypesForDossier( - dossierId, - currentVersion, - true); + List typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, + currentVersion, + true) : dictionaryClient.getAllTypesForDossier(dossierId, + currentVersion, + true); + if (CollectionUtils.isNotEmpty(typeResponse)) { String tenantId = TenantContext.getTenantId(); List dictionary = typeResponse.stream() .parallel() - .map(t -> { - - TenantContext.setTenantId(tenantId); - Optional optionalOldModel; - if (dossierId == null) { - var representation = getDossierTemplateDictionary(dossierTemplateId); - optionalOldModel = representation != null ? representation.getDictionary() - .stream() - .filter(f -> f.getType().equals(t.getType())) - .findAny() : Optional.empty(); - } else { - var representation = getDossierDictionary(dossierId); - optionalOldModel = representation != null ? representation.getDictionary() - .stream() - .filter(f -> f.getType().equals(t.getType())) - .findAny() : Optional.empty(); - } - - Set entries = new HashSet<>(); - Set falsePositives = new HashSet<>(); - Set falseRecommendations = new HashSet<>(); - - DictionaryEntries newEntries = mapEntries(t); - - var newValues = newEntries.getEntries() - .stream() - .map(DictionaryEntry::getValue) - .collect(Collectors.toSet()); - var newFalsePositivesValues = newEntries.getFalsePositives() - .stream() - .map(DictionaryEntry::getValue) - .collect(Collectors.toSet()); - var newFalseRecommendationsValues = newEntries.getFalseRecommendations() - .stream() - .map(DictionaryEntry::getValue) - .collect(Collectors.toSet()); - - optionalOldModel.ifPresent(oldDictionaryModel -> { - - }); - if (optionalOldModel.isPresent()) { - var oldModel = optionalOldModel.get(); - if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) { - // add old entries from existing DictionaryModel but exclude lower case representation - entries.addAll(oldModel.getEntries() - .stream() - .filter(f -> !newValues.stream() - .map(s -> s.toLowerCase(Locale.ROOT)) - .toList().contains(f.getValue())) - .toList()); - falsePositives.addAll(oldModel.getFalsePositives() - .stream() - .filter(f -> !newFalsePositivesValues.stream() - .map(s -> s.toLowerCase(Locale.ROOT)) - .toList().contains(f.getValue())) - .toList()); - falseRecommendations.addAll(oldModel.getFalseRecommendations() - .stream() - .filter(f -> !newFalseRecommendationsValues.stream() - .map(s -> s.toLowerCase(Locale.ROOT)) - .toList().contains(f.getValue())) - .toList()); - } else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) { - // add old entries from existing DictionaryModel but exclude upper case representation - entries.addAll(oldModel.getEntries() - .stream() - .filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT))) - .toList()); - falsePositives.addAll(oldModel.getFalsePositives() - .stream() - .filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT))) - .toList()); - falseRecommendations.addAll(oldModel.getFalseRecommendations() - .stream() - .filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT))) - .toList()); - - } else { - // add old entries from existing DictionaryModel - entries.addAll(oldModel.getEntries() - .stream() - .filter(f -> !newValues.contains(f.getValue())) - .toList()); - falsePositives.addAll(oldModel.getFalsePositives() - .stream() - .filter(f -> !newFalsePositivesValues.contains(f.getValue())) - .toList()); - falseRecommendations.addAll(oldModel.getFalseRecommendations() - .stream() - .filter(f -> !newFalseRecommendationsValues.contains(f.getValue())) - .toList()); - } - } - - // Add Increments - entries.addAll(newEntries.getEntries()); - falsePositives.addAll(newEntries.getFalsePositives()); - falseRecommendations.addAll(newEntries.getFalseRecommendations()); - - DictionaryModel dictionaryModel = new DictionaryModel(t.getType(), - t.getRank(), - convertColor(t.getHexColor()), - t.isCaseInsensitive(), - t.isHint(), - entries, - falsePositives, - falseRecommendations, - dossierId != null); - return dictionaryModel; - }) + .map(t -> mapTypeToDictionaryModel(tenantId, t, dossierTemplateId, dossierId)) .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) .collect(Collectors.toList()); @@ -301,43 +186,171 @@ public class DictionaryService { dictionaryRepresentation.setDictionary(dictionary); if (dossierId == null) { - addDictionaryRepresentationForDossierTemplate(dossierTemplateId, dictionaryRepresentation); + dictionaryCacheService.addDictionaryRepresentationForDossierTemplate(dossierTemplateId, dictionaryRepresentation); } else { - addDictionaryRepresentationForDossier(dossierId, dictionaryRepresentation); + dictionaryCacheService.addDictionaryRepresentationForDossier(dossierId, dictionaryRepresentation); } } } catch (FeignException e) { - log.warn("Got some unknown feignException", e); + log.warn("Got some unknown FeignException", e); throw e; } } + private DictionaryModel mapTypeToDictionaryModel(String tenantId, Type type, String dossierTemplateId, String dossierId) { + + TenantContext.setTenantId(tenantId); + Optional optionalOldModel = getExistingDictionaryModel(type.getType(), dossierTemplateId, dossierId); + + DictionaryEntries newEntries = mapEntries(type); + Set combinedEntries = new HashSet<>(newEntries.getEntries()); + Set combinedFalsePositives = new HashSet<>(newEntries.getFalsePositives()); + Set combinedFalseRecommendations = new HashSet<>(newEntries.getFalseRecommendations()); + + optionalOldModel.ifPresent(oldModel -> handleOldEntries(oldModel, type, newEntries, combinedEntries, combinedFalsePositives, combinedFalseRecommendations)); + + // Add new entries + combinedEntries.addAll(newEntries.getEntries()); + combinedFalsePositives.addAll(newEntries.getFalsePositives()); + combinedFalseRecommendations.addAll(newEntries.getFalseRecommendations()); + + return new DictionaryModel(type.getType(), + type.getRank(), + convertColor(type.getHexColor()), + type.isCaseInsensitive(), + type.isHint(), + combinedEntries, + combinedFalsePositives, + combinedFalseRecommendations, + type.isDossierDictionaryOnly()); + } + + + private Optional getExistingDictionaryModel(String type, String dossierTemplateId, String dossierId) { + + DictionaryRepresentation representation = dossierId == null ? // + dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId) : dictionaryCacheService.getDossierDictionary(dossierId); + + return representation != null ? representation.getDictionary() + .stream() + .filter(f -> f.getType().equals(type)) + .findAny() : Optional.empty(); + } + + + private void handleOldEntries(DictionaryModel oldModel, + Type newType, + DictionaryEntries newEntries, + Set combinedEntries, + Set combinedFalsePositives, + Set combinedFalseRecommendations) { + + if (oldModel.isCaseInsensitive() && !newType.isCaseInsensitive()) { + combinedEntries.addAll(oldModel.getEntries() + .stream() + .filter(f -> !newEntries.getEntries() + .stream() + .map(s -> s.getValue().toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()).contains(f.getValue())) + .collect(Collectors.toSet())); + + combinedFalsePositives.addAll(oldModel.getFalsePositives() + .stream() + .filter(f -> !newEntries.getFalsePositives() + .stream() + .map(s -> s.getValue().toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()).contains(f.getValue())) + .collect(Collectors.toSet())); + + combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations() + .stream() + .filter(f -> !newEntries.getFalseRecommendations() + .stream() + .map(s -> s.getValue().toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()).contains(f.getValue())) + .collect(Collectors.toSet())); + } else if (!oldModel.isCaseInsensitive() && newType.isCaseInsensitive()) { + combinedEntries.addAll(oldModel.getEntries() + .stream() + .filter(f -> !newEntries.getEntries() + .stream() + .map(s -> s.getValue().toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT))) + .collect(Collectors.toSet())); + + combinedFalsePositives.addAll(oldModel.getFalsePositives() + .stream() + .filter(f -> !newEntries.getFalsePositives() + .stream() + .map(s -> s.getValue().toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT))) + .collect(Collectors.toSet())); + + combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations() + .stream() + .filter(f -> !newEntries.getFalseRecommendations() + .stream() + .map(s -> s.getValue().toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT))) + .collect(Collectors.toSet())); + } else { + combinedEntries.addAll(oldModel.getEntries() + .stream() + .filter(f -> !newEntries.getEntries() + .stream() + .map(DictionaryEntryModel::getValue) + .collect(Collectors.toSet()).contains(f.getValue())) + .collect(Collectors.toSet())); + + combinedFalsePositives.addAll(oldModel.getFalsePositives() + .stream() + .filter(f -> !newEntries.getFalsePositives() + .stream() + .map(DictionaryEntryModel::getValue) + .collect(Collectors.toSet()).contains(f.getValue())) + .collect(Collectors.toSet())); + + combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations() + .stream() + .filter(f -> !newEntries.getFalseRecommendations() + .stream() + .map(DictionaryEntryModel::getValue) + .collect(Collectors.toSet()).contains(f.getValue())) + .collect(Collectors.toSet())); + } + } + + private DictionaryEntries mapEntries(Type type) { - Set entries = type.getEntries() != null ? new HashSet<>(type.getEntries() - .stream() - .map(DictionaryEntryModel::new) - .collect(Collectors.toSet())) : new HashSet<>(); - Set falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries() - .stream() - .map(DictionaryEntryModel::new) - .collect(Collectors.toSet())) : new HashSet<>(); - Set falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries() - .stream() - .map(DictionaryEntryModel::new) - .collect(Collectors.toSet())) : new HashSet<>(); + Set entries = type.getEntries() != null ? type.getEntries() + .stream() + .map(DictionaryEntryModel::new) + .collect(Collectors.toSet()) : new HashSet<>(); + + Set falsePositives = type.getFalsePositiveEntries() != null ? type.getFalsePositiveEntries() + .stream() + .map(DictionaryEntryModel::new) + .collect(Collectors.toSet()) : new HashSet<>(); + + Set falseRecommendations = type.getFalseRecommendationEntries() != null ? type.getFalseRecommendationEntries() + .stream() + .map(DictionaryEntryModel::new) + .collect(Collectors.toSet()) : new HashSet<>(); if (type.isCaseInsensitive()) { entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); falsePositives.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); } - log.debug("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}", + + log.debug("Dictionary update returned {} entries, {} falsePositives, and {} falseRecommendations for type {}", entries.size(), falsePositives.size(), falseRecommendations.size(), - entries); + type.getType()); + return new DictionaryEntries(entries, falsePositives, falseRecommendations); } @@ -352,138 +365,69 @@ public class DictionaryService { @SneakyThrows public float[] getColor(String type, String dossierTemplateId) { - DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap() + DictionaryRepresentation dossierTemplateDictionary = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId); + DictionaryModel model = dossierTemplateDictionary.getLocalAccessMap() .get(type); - if (model != null) { - return model.getColor(); - } - return getDossierTemplateDictionary(dossierTemplateId).getDefaultColor(); + return model != null ? model.getColor() : dossierTemplateDictionary.getDefaultColor(); } @SneakyThrows public boolean isHint(String type, String dossierTemplateId) { - DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap() + DictionaryModel model = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap() .get(type); - if (model != null) { - return model.isHint(); - } - return false; - } - - - @SneakyThrows - @Timed("redactmanager_getDeepCopyDictionary") - @Observed(name = "DictionaryService", contextualName = "deep-copy-dictionary") - public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) { - - List mergedDictionaries = new LinkedList<>(); - - DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId); - List dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary(); - dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone())); - - // Add dossier - long dossierDictionaryVersion = -1; - if (dossierDictionaryExists(dossierId)) { - DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId); - List dossierDictionaries = dossierRepresentation.getDictionary(); - dossierDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone())); - return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierRepresentation.getDictionaryVersion()); - } else { - return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierDictionaryVersion); - } - } - - - private Dictionary getDictionary(List mergedDictionaries, DictionaryRepresentation dossierTemplateRepresentation, long dossierDictionaryVersion) { - - // todo: we need caching here ? --> do not create a new one always, so we do not have as many creations of the searchImplementation - // or we add it to DictionaryRepresentation and add it here as well? - // maybe redis as well? - System.out.println("TEST ---> getDictionary"); - DictionaryVersion dictionaryVersion = DictionaryVersion.builder() - .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()) - .dossierVersion(dossierDictionaryVersion) - .build(); - String tenantId = TenantContext.getTenantId(); - - Dictionary cachedDictionary = dictionaryCache.get(tenantId); - - if (cachedDictionary != null) { - DictionaryVersion cachedVersion = cachedDictionary.getVersion(); - - boolean isUpToDate = (dictionaryVersion.getDossierTemplateVersion() <= cachedVersion.getDossierTemplateVersion()) - && (dictionaryVersion.getDossierVersion() <= cachedVersion.getDossierVersion()); - - if (isUpToDate) { - System.out.println("TEST ---> cached Dictionary "); - return cachedDictionary; - } - } - - Dictionary dictionary = new Dictionary(mergedDictionaries.stream() - .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) - .collect(Collectors.toList()), dictionaryVersion); - dictionaryCache.put(tenantId, dictionary); - - return dictionary; - + return model != null && model.isHint(); } @SneakyThrows public float[] getNotRedactedColor(String dossierTemplateId) { - return getDossierTemplateDictionary(dossierTemplateId).getNotRedactedColor(); + return dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId).getNotRedactedColor(); } - @SneakyThrows - private void addDictionaryRepresentationForDossierTemplate(String dossierTemplateId, DictionaryRepresentation dictionaryRepresentation) { + private Dictionary buildDictionary(String dossierTemplateId, String dossierId, DictionaryVersion dictionaryVersion) { - tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation); + List mergedDictionaries = new LinkedList<>(); + + DictionaryRepresentation templateDictRep = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId); + if (templateDictRep != null) { + templateDictRep.getDictionary() + .forEach(dm -> mergedDictionaries.add(dm.clone())); + } + + if (dossierDictionaryExists(dossierId)) { + DictionaryRepresentation dossierDictRep = dictionaryCacheService.getDossierDictionary(dossierId); + if (dossierDictRep != null) { + dossierDictRep.getDictionary() + .forEach(dm -> mergedDictionaries.add(dm.clone())); + } + } + + return dictionaryFactory.create(mergedDictionaries.stream() + .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) + .collect(Collectors.toList()), dictionaryVersion); } - @SneakyThrows - private void addDictionaryRepresentationForDossier(String dossierId, DictionaryRepresentation dictionaryRepresentation) { + private boolean checkIfDictionaryIsUpToDate(String dossierTemplateId, String dossierId, Dictionary cachedDictionary) { - tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation); - } + long latestTemplateVersion = dictionaryClient.getVersion(dossierTemplateId); + long latestDossierVersion = dictionaryClient.getVersionForDossier(dossierId); + DictionaryVersion cachedVersion = cachedDictionary.getVersion(); - @SneakyThrows - private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) { - - return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate() - .get(dossierTemplateId); - } - - - @SneakyThrows - private DictionaryRepresentation getDossierDictionary(String dossierId) { - - return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier() - .get(dossierId); + return (cachedVersion.getDossierTemplateVersion() >= latestTemplateVersion) && (cachedVersion.getDossierVersion() >= latestDossierVersion); } @SneakyThrows private boolean dossierDictionaryExists(String dossierId) { - return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId); - } - - - private Long getVersion(DictionaryRepresentation dictionaryRepresentation) { - - if (dictionaryRepresentation == null) { - return null; - } else { - return dictionaryRepresentation.getDictionaryVersion(); - } + DictionaryRepresentation dossierDictRep = dictionaryCacheService.getDossierDictionary(dossierId); + return dossierDictRep != null; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 7c17a0ee..1538c295 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -265,7 +265,7 @@ public class RedactionStorageService { // And the cache eviction logic when a file changes after e.g. ocr is not implemented yet. // See https://knecon.atlassian.net/jira/software/c/projects/RED/boards/37?selectedIssue=RED-8106. @Timed("redactmanager_getDocumentGraph") - @Cacheable(value = "documentDataCache") + //@Cacheable(value = "documentDataCache") public DocumentData getDocumentData(String dossierId, String fileId) { try { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 9900f7ee..1d015c89 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -56,6 +56,7 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; +import com.iqser.red.service.redaction.v1.server.service.DictionaryCacheService; import com.iqser.red.service.redaction.v1.server.service.DocumentSearchService; import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService; import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService; @@ -196,6 +197,10 @@ public abstract class AbstractRedactionIntegrationTest { @Autowired protected TenantMongoLiquibaseExecutor tenantMongoLiquibaseExecutor; + @Autowired + DictionaryCacheService dictionaryCacheService; + + protected final Map> dictionary = new HashMap<>(); protected final Map> dossierDictionary = new HashMap<>(); protected final Map> falsePositive = new HashMap<>(); @@ -260,6 +265,7 @@ public abstract class AbstractRedactionIntegrationTest { } entityLogDocumentRepository.deleteAll(); entityLogEntryDocumentRepository.deleteAll(); + dictionaryCacheService.clearAllCaches(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index f4a3e36b..cc26d8e2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -62,6 +62,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryFactory; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; @@ -148,6 +149,8 @@ import lombok.extern.slf4j.Slf4j; private MongoConnectionProvider mongoConnectionProvider; @MockBean private TenantProvider tenantProvider; + @Autowired + private DictionaryFactory dictionaryFactory; @Test @@ -250,7 +253,7 @@ import lombok.extern.slf4j.Slf4j; testDossierTemplate = new TestDossierTemplate(dossierTemplateToUse); when(dictionaryService.updateDictionary(any(), any())).thenReturn(new DictionaryVersion(0, 0)); - when(dictionaryService.getDeepCopyDictionary(any(), any())).thenReturn(testDossierTemplate.testDictionary); + when(dictionaryService.getDictionary(any(), any())).thenReturn(testDossierTemplate.testDictionary); when(dictionaryService.getDictionaryIncrements(any(), any(), any())).thenReturn(new DictionaryIncrement(Collections.emptySet(), new DictionaryVersion(0, 0))); when(dictionaryService.isHint(any(String.class), any())).thenAnswer(invocation -> { String type = invocation.getArgument(0); @@ -422,7 +425,7 @@ import lombok.extern.slf4j.Slf4j; componentRules = new String(Files.readAllBytes(componentRuleFile.toPath())); } - testDictionary = new Dictionary(dictionaries, new DictionaryVersion(0, 0)); + testDictionary = dictionaryFactory.create(dictionaries, new DictionaryVersion(0, 0)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index b92b6ae2..9f102b0a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -252,14 +252,14 @@ public class DictionaryServiceTest { when(dictionaryClient.getDictionaryForType("dossierType", 0L)).thenReturn(dossierType); dictionaryService.updateDictionary("dtId", "dossierId"); - var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId"); - assertThat(dict.getDictionaryModels().size()).isEqualTo(2); + var dict = dictionaryService.getDictionary("dtId", "dossierId"); + assertThat(dict.getDictionaryModels().size()).isEqualTo(1); var dictModel = dict.getDictionaryModels() .get(0); assertThat(dictModel.getType()).isEqualTo(type); assertThat(dictModel.getEntries().size()).isEqualTo(3); dictModel.getEntries() - .forEach(entry -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId())); + .forEach(entry -> assertThat(entry.getTypeId()).isEqualTo("dossierType")); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 9e69bd34..14552180 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -45,10 +45,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.service.redaction.v1.server.service.DictionaryCacheService; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.StorageAutoConfiguration; @@ -70,6 +70,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { @Autowired DictionaryService dictionaryService; + @Autowired + DictionaryCacheService dictionaryCacheService; + @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @Import(LayoutParsingServiceProcessorConfiguration.class) @@ -136,7 +139,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>()); - dictionaryService.clearTenantDictionaryCache(); + dictionaryCacheService.clearAllCaches(); AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf"); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java index 02fbb2ea..f91c9758 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java @@ -14,31 +14,35 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch; +import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch; +import com.iqser.red.service.redaction.v1.server.model.dictionary.HyperscanDictionarySearch; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.gliwka.hyperscan.wrapper.CompileErrorException; @Disabled public class DictionaryPerformanceTest { - private static final int LARGE_DICTIONARY_SIZE = 6_000; - private static final int LARGE_TEXT_REPETITIONS = 500_000; + private static final int LARGE_DICTIONARY_SIZE = 5_000; + private static final int LARGE_TEXT_REPETITIONS = 50_000; private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " + "Entity_1 match text. Recommendation_1 also here. Random text continues. "; - @Test public void testLargeFileWithLargeDictionaries() { Map> dictionaryValues = new HashMap<>(); + // Generate dictionary values with specific terms for matching IntStream.range(0, 6) .forEach(i -> { EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION; boolean caseSensitive = i % 2 == 0; - DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive); + DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, + caseSensitive); List dictionary = generateLargeDictionary(); // Add specific terms that are included in the large text for matches @@ -51,23 +55,36 @@ public class DictionaryPerformanceTest { dictionaryValues.put(identifier, dictionary); }); + // Measure construction time for DictionarySearchImplementation long dictionaryTrieConstructionStart = System.currentTimeMillis(); - DictionarySearchImplementation dictionarySearchImpl = new DictionarySearchImplementation(dictionaryValues); + TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(dictionaryValues); long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart; + // Measure construction time for SearchImplementations long searchTrieConstructionStart = System.currentTimeMillis(); - List searchImplementations = dictionaryValues.entrySet() - .stream() - .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive())) - .toList(); + List searchImplementations = dictionaryValues.entrySet().stream() + .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive())).toList(); long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart; + // Measure construction time for HyperSearchImplementation + long hyperSearchConstructionStart = System.currentTimeMillis(); + HyperscanDictionarySearch hyperSearchImpl = null; + try { + hyperSearchImpl = new HyperscanDictionarySearch(dictionaryValues); + } catch (CompileErrorException e) { + assert false : "Failed to compile HyperSearchImplementation: " + e.getMessage(); + } + long hyperSearchConstructionDuration = System.currentTimeMillis() - hyperSearchConstructionStart; + String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS); + // Measure search time for DictionarySearchImplementation long dictionarySearchStart = System.currentTimeMillis(); - List dictionaryMatches = dictionarySearchImpl.getBoundaries(largeText); + List dictionaryMatches = dictionarySearchImpl + .getBoundariesAsList(largeText); long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart; + // Measure search time for SearchImplementations long searchImplStart = System.currentTimeMillis(); List searchMatches = new ArrayList<>(); for (SearchImplementation searchImpl : searchImplementations) { @@ -75,30 +92,46 @@ public class DictionaryPerformanceTest { } long searchImplDuration = System.currentTimeMillis() - searchImplStart; + // Measure search time for HyperSearchImplementation + long hyperSearchStart = System.currentTimeMillis(); + List hyperMatches = hyperSearchImpl.getMatchesAsList(largeText); + long hyperSearchDuration = System.currentTimeMillis() - hyperSearchStart; + System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration); - System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration, dictionaryMatches.size()); + System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration, + dictionaryMatches.size()); System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration); - System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration, searchMatches.size()); + System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration, + searchMatches.size()); + System.out.printf("HyperSearchImplementation construction took %d ms\n", hyperSearchConstructionDuration); + System.out.printf("HyperSearchImplementation search took %d ms and found %d matches\n", hyperSearchDuration, + hyperMatches.size()); - assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() : "Both implementations should find entities."; - assertEquals("Both implementations should find the same number of matches", dictionaryMatches.size(), searchMatches.size()); + assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() && !hyperMatches.isEmpty() + : "All implementations should find entities."; + + // Ensure all implementations found the same number of matches + assertEquals("DictionarySearchImplementation and SearchImplementations should find the same number of matches", + dictionaryMatches.size(), searchMatches.size()); + assertEquals("DictionarySearchImplementation and HyperSearchImplementation should find the same number of matches", + dictionaryMatches.size(), hyperMatches.size()); + + // Close the HyperSearchImplementation to release resources + hyperSearchImpl.close(); } - private List generateLargeDictionary() { - - return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString()) + return IntStream.range(0, LARGE_DICTIONARY_SIZE) + .mapToObj(i -> RandomStringGenerator.generateRandomString()) .collect(Collectors.toList()); } - static final class RandomStringGenerator { private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; private static final int STRING_LENGTH = 50; private static final SecureRandom RANDOM = new SecureRandom(); - public static String generateRandomString() { StringBuilder sb = new StringBuilder(STRING_LENGTH); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index c49e4264..3f86916d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -36,7 +36,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch; +import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; @@ -125,7 +126,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration Document document = buildGraph(filename); dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); + Dictionary dictionary = dictionaryService.getDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); long dictionarySearchStart = System.currentTimeMillis(); List foundEntities = new LinkedList<>(); @@ -196,7 +197,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration Document document = buildGraph(filename); dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); + Dictionary dictionary = dictionaryService.getDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); int numberOfRuns = 1; float totalSearchTime = 0; @@ -290,10 +291,10 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration } - private void findEntitiesWithSearchImplementation(Document document, DictionarySearchImplementation dictionarySearchImplementation, List foundEntities) { + private void findEntitiesWithSearchImplementation(Document document, DictionarySearch dictionarySearch, List foundEntities) { TextBlock textBlock = document.getTextBlock(); - dictionarySearchImplementation.getBoundaries(textBlock) + dictionarySearch.getBoundaries(textBlock) .filter(match -> boundaryIsSurroundedBySeparators(textBlock, match.textRange())) .map(match -> TextEntity.initialEntityNode(match.textRange(), match.identifier().type(), match.identifier().entityType(), document)) .forEach(foundEntities::add); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java index f2cfea78..7d5d61ff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java @@ -246,7 +246,7 @@ public class LiveDataIntegrationTest { dictionaryService.updateDictionary("dossierTemplateId", "dossierId"); - var dict = dictionaryService.getDeepCopyDictionary("dossierTemplateId", "dossierId"); + var dict = dictionaryService.getDictionary("dossierTemplateId", "dossierId"); assertThat(dict.getLocalAccessMap().size()).isEqualTo(12); }