Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-10290: Improve SearchImplementation logic for dictionaries

Browse Source
This commit is contained in:
maverickstuder 2024-10-28 16:19:57 +01:00
parent 62aa7d9187
commit c93e3812b6
19 changed files with 872 additions and 485 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -87,6 +87,7 @@ dependencies {
implementation("org.reflections:reflections:0.10.2") implementation("org.reflections:reflections:0.10.2")
implementation("com.opencsv:opencsv:5.9") implementation("com.opencsv:opencsv:5.9")
implementation("com.gliwka.hyperscan:hyperscan:5.4.11-3.0.0")
implementation("com.joestelmach:natty:0.13") implementation("com.joestelmach:natty:0.13")
testImplementation(project(":rules-management")) testImplementation(project(":rules-management"))

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java
View File

@ -28,6 +28,8 @@ public class RedactionServiceSettings {
private boolean priorityMode; private boolean priorityMode;
private long firstLevelDictionaryCacheMaximumSize = 1000;
private long dictionaryCacheMaximumSize = 100; private long dictionaryCacheMaximumSize = 100;
private int dictionaryCacheExpireAfterAccessDays = 3; private int dictionaryCacheExpireAfterAccessDays = 3;

100
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java
View File

@ -24,33 +24,26 @@ import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundExcepti
import lombok.Data; import lombok.Data;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor;
/** /**
* A class representing a dictionary used for redaction processes, containing various dictionary models and their versions. * A class representing a dictionary used for redaction processes, containing various dictionary models and their versions.
*/ */
@Data @Data
@NoArgsConstructor
public class Dictionary { public class Dictionary {
@Getter private final Map<String, DictionaryModel> localAccessMap = new HashMap<>();
private List<DictionaryModel> dictionaryModels;
// todo: dossier and dossier template level DictionaryModels override each other
// at the moment there are no problems because they always have the same rank / hint information
// but it should be changed so that the localAccessMap contains all models
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
@Getter @Getter
private DictionaryVersion version; private final DictionaryVersion version;
private DictionarySearchImplementation dictionarySearch; private final DictionarySearch dictionarySearch;
public Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version) { Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version, DictionarySearch dictionarySearch) {
this.dictionaryModels = dictionaryModels; dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
this.version = version; this.version = version;
this.dictionarySearch = dictionarySearch;
} }
@ -70,11 +63,19 @@ public class Dictionary {
*/ */
public boolean hasLocalEntries() { public boolean hasLocalEntries() {
return dictionaryModels.stream() return getDictionaryModels().stream()
.anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty()); .anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty());
} }
public List<DictionaryModel> getDictionaryModels() {
return localAccessMap.values()
.stream()
.toList();
}
public Set<String> getTypes() { public Set<String> getTypes() {
return localAccessMap.keySet(); return localAccessMap.keySet();
@ -130,77 +131,6 @@ public class Dictionary {
} }
public DictionarySearchImplementation getDictionarySearch() {
if (dictionarySearch == null) {
dictionarySearch = new DictionarySearchImplementation(generateCombinedDictionaryValuesMap());
}
return dictionarySearch;
}
private Map<DictionaryIdentifier, List<String>> generateCombinedDictionaryValuesMap() {
Map<DictionaryIdentifier, List<String>> combinedValuesMap = new HashMap<>();
for (DictionaryModel model : dictionaryModels) {
DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive());
DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(),
EntityType.DICTIONARY_REMOVAL,
model.isDossierDictionary(),
!model.isCaseInsensitive());
DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(),
EntityType.FALSE_POSITIVE,
model.isDossierDictionary(),
!model.isCaseInsensitive());
DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(),
EntityType.FALSE_RECOMMENDATION,
model.isDossierDictionary(),
!model.isCaseInsensitive());
List<String> entryValues = model.getEntries()
.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.toList();
List<String> deletedEntryValues = model.getEntries()
.stream()
.filter(DictionaryEntry::isDeleted)
.map(DictionaryEntry::getValue)
.toList();
List<String> falsePositiveValues = model.getFalsePositives()
.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.toList();
List<String> falseRecommendationValues = model.getFalseRecommendations()
.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.toList();
if (!entryValues.isEmpty()) {
combinedValuesMap.put(entriesIdentifier, entryValues);
}
if (!deletedEntryValues.isEmpty()) {
combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues);
}
if (!falsePositiveValues.isEmpty()) {
combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues);
}
if (!falseRecommendationValues.isEmpty()) {
combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues);
}
}
return combinedValuesMap;
}
/** /**
* Adds a local dictionary entry of a specific type. * Adds a local dictionary entry of a specific type.
* *

103
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryFactory.java Normal file
View File

@ -0,0 +1,103 @@
package com.iqser.red.service.redaction.v1.server.model.dictionary;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.stereotype.Service;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@Service
@RequiredArgsConstructor
public class DictionaryFactory {
private final RedactionServiceSettings settings;
@SneakyThrows
public Dictionary create(List<DictionaryModel> dictionaryModels, DictionaryVersion dictionaryVersion) {
// Generate the combined dictionary values map
Map<DictionaryIdentifier, List<String>> combinedMap = generateCombinedDictionaryValuesMap(dictionaryModels);
// Determine which DictionarySearch implementation to use
DictionarySearch dictionarySearch;
if (settings.isPriorityMode()) {
dictionarySearch = new HyperscanDictionarySearch(combinedMap);
} else {
dictionarySearch = new TrieDictionarySearch(combinedMap);
}
// Instantiate and return the Dictionary
return new Dictionary(dictionaryModels, dictionaryVersion, dictionarySearch);
}
private Map<DictionaryIdentifier, List<String>> generateCombinedDictionaryValuesMap(List<DictionaryModel> dictionaryModels) {
Map<DictionaryIdentifier, List<String>> combinedValuesMap = new HashMap<>();
for (DictionaryModel model : dictionaryModels) {
DictionaryIdentifier entriesIdentifier = new DictionaryIdentifier(model.getType(), EntityType.ENTITY, model.isDossierDictionary(), !model.isCaseInsensitive());
DictionaryIdentifier deletedEntriesIdentifier = new DictionaryIdentifier(model.getType(),
EntityType.DICTIONARY_REMOVAL,
model.isDossierDictionary(),
!model.isCaseInsensitive());
DictionaryIdentifier falsePositiveIdentifier = new DictionaryIdentifier(model.getType(),
EntityType.FALSE_POSITIVE,
model.isDossierDictionary(),
!model.isCaseInsensitive());
DictionaryIdentifier falseRecommendationsIdentifier = new DictionaryIdentifier(model.getType(),
EntityType.FALSE_RECOMMENDATION,
model.isDossierDictionary(),
!model.isCaseInsensitive());
List<String> entryValues = model.getEntries()
.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.toList();
List<String> deletedEntryValues = model.getEntries()
.stream()
.filter(DictionaryEntry::isDeleted)
.map(DictionaryEntry::getValue)
.toList();
List<String> falsePositiveValues = model.getFalsePositives()
.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.toList();
List<String> falseRecommendationValues = model.getFalseRecommendations()
.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.toList();
if (!entryValues.isEmpty()) {
combinedValuesMap.put(entriesIdentifier, entryValues);
}
if (!deletedEntryValues.isEmpty()) {
combinedValuesMap.put(deletedEntriesIdentifier, deletedEntryValues);
}
if (!falsePositiveValues.isEmpty()) {
combinedValuesMap.put(falsePositiveIdentifier, falsePositiveValues);
}
if (!falseRecommendationValues.isEmpty()) {
combinedValuesMap.put(falseRecommendationsIdentifier, falseRecommendationValues);
}
}
return combinedValuesMap;
}
}

94
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearch.java Normal file
View File

@ -0,0 +1,94 @@
package com.iqser.red.service.redaction.v1.server.model.dictionary;
import java.util.List;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
/**
* Common interface for dictionary search implementations.
*/
public interface DictionarySearch {
/**
* Checks if at least one dictionary entry matches the given text.
*
* @param text The text to search within.
* @return True if at least one match is found, otherwise false.
*/
boolean atLeastOneMatches(String text);
/**
* Retrieves a list of match boundaries within the given text.
*
* @param text The text to search within.
* @return A list of MatchTextRange representing the boundaries of matches.
*/
default List<MatchTextRange> getBoundariesAsList(CharSequence text) {
return getBoundaries(text).toList();
}
/**
* Retrieves a stream of match boundaries within the given text.
*
* @param text The text to search within.
* @return A stream of MatchTextRange representing the boundaries of matches.
*/
Stream<MatchTextRange> getBoundaries(CharSequence text);
/**
* Retrieves a list of match boundaries within a specified region of the text.
*
* @param text The text to search within.
* @param region The specific region of the text to search.
* @return A list of MatchTextRange representing the boundaries of matches.
*/
default List<MatchTextRange> getBoundariesAsList(CharSequence text, TextRange region) {
return getBoundaries(text, region).toList();
}
/**
* Retrieves a stream of match boundaries within a specified region of the text.
*
* @param text The text to search within.
* @param region The specific region of the text to search.
* @return A stream of MatchTextRange representing the boundaries of matches.
*/
Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region);
/**
* Retrieves a stream of match boundaries within the given TextBlock.
*
* @param textBlock The TextBlock to search within.
* @return A stream of MatchTextRange representing the boundaries of matches.
*/
Stream<MatchTextRange> getBoundaries(TextBlock textBlock);
/**
* Retrieves a list of match positions within the given text.
*
* @param text The text to search within.
* @return A list of MatchPosition representing the positions of matches.
*/
default List<MatchPosition> getMatchesAsList(String text) {
return getMatches(text).toList();
}
/**
* Retrieves a stream of match positions within the given text.
*
* @param text The text to search within.
* @return A stream of MatchPosition representing the positions of matches.
*/
Stream<MatchPosition> getMatches(String text);
/**
* Record representing the range of matched text along with its identifier.
*/
record MatchTextRange(DictionaryIdentifier identifier, TextRange textRange) {}
/**
* Record representing the start and end positions of a match along with its identifier.
*/
record MatchPosition(DictionaryIdentifier identifier, int startIndex, int endIndex) {}
}

172
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/HyperscanDictionarySearch.java Normal file
View File

@ -0,0 +1,172 @@
package com.iqser.red.service.redaction.v1.server.model.dictionary;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import com.gliwka.hyperscan.wrapper.CompileErrorException;
import com.gliwka.hyperscan.wrapper.Database;
import com.gliwka.hyperscan.wrapper.Expression;
import com.gliwka.hyperscan.wrapper.ExpressionFlag;
import com.gliwka.hyperscan.wrapper.Match;
import com.gliwka.hyperscan.wrapper.Scanner;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
public class HyperscanDictionarySearch implements DictionarySearch {
private final Map<Integer, DictionaryIdentifier> patternIdMap = new ConcurrentHashMap<>();
private AtomicInteger patternIdCounter = new AtomicInteger(0);
private Database database;
public HyperscanDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) throws CompileErrorException {
Map<DictionaryIdentifier, List<String>> entries = new HashMap<>(dictionaryValues);
this.database = compileDatabase(entries, this.patternIdMap);
}
private Database compileDatabase(Map<DictionaryIdentifier, List<String>> entries, Map<Integer, DictionaryIdentifier> patternIdMap) throws CompileErrorException {
if (entries.isEmpty()) {
return null;
}
LinkedList<Expression> expressions = new LinkedList<>();
for (Map.Entry<DictionaryIdentifier, List<String>> entry : entries.entrySet()) {
DictionaryIdentifier identifier = entry.getKey();
for (String value : entry.getValue()) {
int patternId = patternIdCounter.incrementAndGet();
patternIdMap.put(patternId, identifier);
EnumSet<ExpressionFlag> flags = EnumSet.noneOf(ExpressionFlag.class);
if (!identifier.caseSensitive()) {
flags.add(ExpressionFlag.CASELESS);
}
// escape regex characters as we only use literals
expressions.add(new Expression(Pattern.quote(value), flags, patternId));
}
}
// Build the database
return Database.compile(expressions);
}
@Override
public boolean atLeastOneMatches(String text) {
if (this.database == null) {
return false;
}
try (Scanner scanner = new Scanner()) {
scanner.allocScratch(this.database);
List<Match> matches = scanner.scan(this.database, text);
return !matches.isEmpty();
}
}
@Override
public Stream<MatchTextRange> getBoundaries(CharSequence text) {
List<MatchTextRange> boundaries = new ArrayList<>();
if (this.database == null) {
return Stream.empty();
}
try (Scanner scanner = new Scanner()) {
scanner.allocScratch(this.database);
List<Match> matches = scanner.scan(this.database, text.toString());
for (Match match : matches) {
int start = (int) match.getStartPosition();
int end = (int) match.getEndPosition();
int patternId = match.getMatchedExpression().getId();
DictionaryIdentifier identifier = patternIdMap.get(patternId);
boundaries.add(new MatchTextRange(identifier, new TextRange(start, end)));
}
}
return boundaries.stream();
}
@Override
public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
return getBoundaries(textBlock, textBlock.getTextRange());
}
@Override
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
List<MatchTextRange> boundaries = new ArrayList<>();
if (this.database == null) {
return Stream.empty();
}
CharSequence subText = text.subSequence(region.start(), region.end());
try (Scanner scanner = new Scanner()) {
scanner.allocScratch(this.database);
List<Match> matches = scanner.scan(this.database, subText.toString());
for (Match match : matches) {
int start = (int) (match.getStartPosition() + region.start());
int end = (int) (match.getEndPosition() + region.start());
int patternId = match.getMatchedExpression().getId();
DictionaryIdentifier identifier = patternIdMap.get(patternId);
boundaries.add(new MatchTextRange(identifier, new TextRange(start, end)));
}
}
return boundaries.stream();
}
@Override
public Stream<MatchPosition> getMatches(String text) {
List<MatchPosition> matchPositions = new ArrayList<>();
if (this.database == null) {
return Stream.empty();
}
try (Scanner scanner = new Scanner()) {
scanner.allocScratch(this.database);
List<Match> matches = scanner.scan(this.database, text);
for (Match match : matches) {
int start = (int) match.getStartPosition();
int end = (int) match.getEndPosition();
int patternId = match.getMatchedExpression().getId();
DictionaryIdentifier identifier = patternIdMap.get(patternId);
matchPositions.add(new MatchPosition(identifier, start, end));
}
}
return matchPositions.stream();
}
public void close() {
if (this.database != null) {
this.database.close();
this.database = null;
}
}
}

75
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionarySearchImplementation.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TrieDictionarySearch.java
View File

@ -4,20 +4,21 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
public class DictionarySearchImplementation { public class TrieDictionarySearch implements DictionarySearch {
private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new HashMap<>(); private final Map<DictionaryIdentifier, List<String>> caseSensitiveEntries = new ConcurrentHashMap<>();
private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new HashMap<>(); private final Map<DictionaryIdentifier, List<String>> caseInsensitiveEntries = new ConcurrentHashMap<>();
private final DictionaryTrie caseSensitiveTrie; private final DictionaryTrie caseSensitiveTrie;
private final DictionaryTrie caseInsensitiveTrie; private final DictionaryTrie caseInsensitiveTrie;
public DictionarySearchImplementation(Map<DictionaryIdentifier, List<String>> dictionaryValues) { public TrieDictionarySearch(Map<DictionaryIdentifier, List<String>> dictionaryValues) {
System.out.println("TEST ---> DictionarySearchImplementation CTOR"); System.out.println("TEST ---> DictionarySearchImplementation CTOR");
for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) { for (Map.Entry<DictionaryIdentifier, List<String>> entry : dictionaryValues.entrySet()) {
@ -53,6 +54,7 @@ public class DictionarySearchImplementation {
} }
@Override
public boolean atLeastOneMatches(String text) { public boolean atLeastOneMatches(String text) {
if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) { if (!caseSensitiveEntries.isEmpty() && caseSensitiveTrie != null && caseSensitiveTrie.containsMatch(text)) {
@ -62,12 +64,40 @@ public class DictionarySearchImplementation {
} }
public List<MatchTextRange> getBoundaries(CharSequence text) { @Override
public Stream<MatchTextRange> getBoundaries(CharSequence text) {
List<MatchTextRange> matches = new ArrayList<>(); List<MatchTextRange> matches = new ArrayList<>();
addMatchTextRangesForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text); addMatchTextRangesForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text);
addMatchTextRangesForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text); addMatchTextRangesForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text);
return matches; return matches.stream();
}
@Override
public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
return getBoundaries(textBlock, textBlock.getTextRange());
}
@Override
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
List<MatchTextRange> matches = new ArrayList<>();
addMatchTextRangesForTrie(text, region, matches, caseSensitiveEntries, caseSensitiveTrie);
addMatchTextRangesForTrie(text, region, matches, caseInsensitiveEntries, caseInsensitiveTrie);
return matches.stream();
}
@Override
public Stream<MatchPosition> getMatches(String text) {
List<MatchPosition> matches = new ArrayList<>();
addMatchPositionsForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text);
addMatchPositionsForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text);
return matches.stream();
} }
@ -82,21 +112,6 @@ public class DictionarySearchImplementation {
} }
public Stream<MatchTextRange> getBoundaries(TextBlock textBlock) {
return getBoundaries(textBlock, textBlock.getTextRange());
}
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
List<MatchTextRange> matches = new ArrayList<>();
addMatchTextRangesForTrie(text, region, matches, caseSensitiveEntries, caseSensitiveTrie);
addMatchTextRangesForTrie(text, region, matches, caseInsensitiveEntries, caseInsensitiveTrie);
return matches.stream();
}
private void addMatchTextRangesForTrie(CharSequence text, private void addMatchTextRangesForTrie(CharSequence text,
TextRange region, TextRange region,
List<MatchTextRange> matches, List<MatchTextRange> matches,
@ -113,15 +128,6 @@ public class DictionarySearchImplementation {
} }
public List<MatchPosition> getMatches(String text) {
List<MatchPosition> matches = new ArrayList<>();
addMatchPositionsForTrie(caseSensitiveEntries, caseSensitiveTrie, matches, text);
addMatchPositionsForTrie(caseInsensitiveEntries, caseInsensitiveTrie, matches, text);
return matches;
}
private void addMatchPositionsForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchPosition> matches, String text) { private void addMatchPositionsForTrie(Map<DictionaryIdentifier, List<String>> entries, DictionaryTrie trie, List<MatchPosition> matches, String text) {
if (!entries.isEmpty() && trie != null) { if (!entries.isEmpty() && trie != null) {
@ -132,13 +138,4 @@ public class DictionarySearchImplementation {
} }
} }
public record MatchTextRange(DictionaryIdentifier identifier, TextRange textRange) {
}
public record MatchPosition(DictionaryIdentifier identifier, int startIndex, int endIndex) {
}
} }

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java
View File

@ -8,7 +8,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.function.Supplier; import java.util.function.Supplier;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -102,7 +101,7 @@ public class AnalysisPreparationService {
CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join(); CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join();
Dictionary dictionary = getDictionary(analyzeRequest); Dictionary dictionary = dictionaryService.getDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Document document = documentFuture.get(); Document document = documentFuture.get();
ImportedRedactions importedRedactions = importedRedactionsFuture.get(); ImportedRedactions importedRedactions = importedRedactionsFuture.get();
@ -193,7 +192,7 @@ public class AnalysisPreparationService {
taskExecutor); taskExecutor);
CompletableFuture<DictionaryAndNotFoundEntries> dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> { CompletableFuture<DictionaryAndNotFoundEntries> dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> {
Dictionary dictionary = getDictionary(analyzeRequest); Dictionary dictionary = dictionaryService.getDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, reanalysisSetupData.document(), reanalysisInitialProcessingData.importedRedactions()); NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, reanalysisSetupData.document(), reanalysisInitialProcessingData.importedRedactions());
return new DictionaryAndNotFoundEntries(dictionary, notFoundEntries.notFoundManualRedactionEntries(), notFoundEntries.notFoundImportedEntries()); return new DictionaryAndNotFoundEntries(dictionary, notFoundEntries.notFoundManualRedactionEntries(), notFoundEntries.notFoundImportedEntries());
}, taskExecutor); }, taskExecutor);
@ -251,15 +250,6 @@ public class AnalysisPreparationService {
} }
private Dictionary getDictionary(AnalyzeRequest analyzeRequest) {
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
return dictionary;
}
private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions) { private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions) {
var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,

111
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryCacheService.java Normal file
View File

@ -0,0 +1,111 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import org.springframework.stereotype.Service;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.model.dictionary.TenantDictionary;
import com.knecon.fforesight.tenantcommons.TenantContext;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class DictionaryCacheService {
private final RedactionServiceSettings settings;
private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
private Cache<DictionaryCacheKey, Dictionary> dictionaryCache;
@PostConstruct
protected void createCache() {
tenantDictionaryCache = CacheBuilder.newBuilder()
.maximumSize(settings.getDictionaryCacheMaximumSize())
.expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS)
.build(new CacheLoader<>() {
public TenantDictionary load(String key) {
return new TenantDictionary();
}
});
dictionaryCache = CacheBuilder.newBuilder()
.maximumSize(settings.getFirstLevelDictionaryCacheMaximumSize())
.expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS)
.build();
}
public void clearAllCaches() {
tenantDictionaryCache.invalidateAll();
dictionaryCache.invalidateAll();
}
@SneakyThrows
public DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate()
.get(dossierTemplateId);
}
@SneakyThrows
public DictionaryRepresentation getDossierDictionary(String dossierId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier()
.get(dossierId);
}
@SneakyThrows
public void addDictionaryRepresentationForDossierTemplate(String dossierTemplateId, DictionaryRepresentation dictionaryRepresentation) {
tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation);
}
@SneakyThrows
public void addDictionaryRepresentationForDossier(String dossierId, DictionaryRepresentation dictionaryRepresentation) {
tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation);
}
@SneakyThrows
public Optional<Dictionary> getDictionary(String tenantId, String dossierId) {
return Optional.ofNullable(dictionaryCache.getIfPresent(new DictionaryCacheKey(tenantId, dossierId)));
}
@SneakyThrows
public void putDictionary(String tenantId, String dossierId, Dictionary newDictionary) {
dictionaryCache.put(new DictionaryCacheKey(tenantId, dossierId), newDictionary);
}
public record DictionaryCacheKey(String tenantId, String dossierId) {
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -7,9 +7,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;

550
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java
View File

@ -8,37 +8,28 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryEntries; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryEntries;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryFactory;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryRepresentation; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.dictionary.TenantDictionary;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import feign.FeignException; import feign.FeignException;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed; import io.micrometer.observation.annotation.Observed;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -50,32 +41,39 @@ public class DictionaryService {
public static final String DEFAULT_COLOR = "#cccccc"; public static final String DEFAULT_COLOR = "#cccccc";
private final DictionaryClient dictionaryClient; private final DictionaryClient dictionaryClient;
private final DictionaryCacheService dictionaryCacheService;
private final RedactionServiceSettings settings; private final DictionaryFactory dictionaryFactory;
private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
private ConcurrentMap<String, Dictionary> dictionaryCache = new ConcurrentHashMap<>();
@PostConstruct @SneakyThrows
protected void createCaches() { @Observed(name = "DictionaryService", contextualName = "get-dictionary")
@Timed("redactmanager_getDictionary")
public Dictionary getDictionary(String dossierTemplateId, String dossierId) {
tenantDictionaryCache = CacheBuilder.newBuilder() String tenantId = TenantContext.getTenantId();
.maximumSize(settings.getDictionaryCacheMaximumSize())
.expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS)
.build(new CacheLoader<>() {
public TenantDictionary load(String key) {
return new TenantDictionary(); Optional<Dictionary> cachedDictionary = dictionaryCacheService.getDictionary(tenantId, dossierId);
if (cachedDictionary.isPresent()) {
log.debug("Dictionary found in cache");
boolean isUpToDate = checkIfDictionaryIsUpToDate(dossierTemplateId, dossierId, cachedDictionary.get());
if (isUpToDate) {
log.info("Returning cached Dictionary for tenantId: {}, dossierId: {}", tenantId, dossierId);
return cachedDictionary.get();
} else {
log.info("Cached Dictionary is outdated for tenantId: {}, dossierId: {}", tenantId, dossierId);
} }
}); } else {
log.info("No cached Dictionary found for tenantId: {}, dossierId: {}", tenantId, dossierId);
} }
DictionaryVersion latestVersion = updateDictionary(dossierTemplateId, dossierId);
Dictionary newDictionary = buildDictionary(dossierTemplateId, dossierId, latestVersion);
public void clearTenantDictionaryCache() { dictionaryCacheService.putDictionary(tenantId, dossierId, newDictionary);
log.info("Cached new Dictionary for tenantId: {}, dossierId: {}", tenantId, dossierId);
tenantDictionaryCache.invalidateAll(); return newDictionary;
} }
@ -85,19 +83,24 @@ public class DictionaryService {
public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) { public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
log.debug("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId); log.debug("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
var dossierTemplateDictionary = getDossierTemplateDictionary(dossierTemplateId); // Update template dictionary
if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) { long latestTemplateVersion = dictionaryClient.getVersion(dossierTemplateId);
updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null); DictionaryRepresentation templateDictRep = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId);
if (templateDictRep == null || latestTemplateVersion > templateDictRep.getDictionaryVersion()) {
updateDictionaryEntry(dossierTemplateId, latestTemplateVersion, templateDictRep != null ? templateDictRep.getDictionaryVersion() : null, null);
} }
long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId); // Update dossier dictionary
var dossierDictionary = getDossierDictionary(dossierId); long latestDossierVersion = dictionaryClient.getVersionForDossier(dossierId);
if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) { DictionaryRepresentation dossierDictRep = dictionaryCacheService.getDossierDictionary(dossierId);
updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
if (dossierDictRep == null || latestDossierVersion > dossierDictRep.getDictionaryVersion()) {
updateDictionaryEntry(dossierTemplateId, latestDossierVersion, dossierDictRep != null ? dossierDictRep.getDictionaryVersion() : null, dossierId);
} }
return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build(); return DictionaryVersion.builder().dossierTemplateVersion(latestTemplateVersion).dossierVersion(latestDossierVersion).build();
} }
@ -106,57 +109,46 @@ public class DictionaryService {
@Timed("redactmanager_getDictionaryIncrements") @Timed("redactmanager_getDictionaryIncrements")
public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) { public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId); DictionaryVersion latestVersion = updateDictionary(dossierTemplateId, dossierId);
Set<DictionaryIncrementValue> newValues = new HashSet<>(); Set<DictionaryIncrementValue> newValues = new HashSet<>();
List<DictionaryModel> dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary(); List<DictionaryModel> templateDictionaries = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> { templateDictionaries.forEach(dictionaryModel -> {
dictionaryModel.getEntries() addNewEntries(dictionaryModel, fromVersion.getDossierTemplateVersion(), newValues);
.forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
dictionaryModel.getFalsePositives()
.forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
dictionaryModel.getFalseRecommendations()
.forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
}); });
if (dossierDictionaryExists(dossierId)) { if (dossierDictionaryExists(dossierId)) {
dictionaryModels = getDossierDictionary(dossierId).getDictionary(); List<DictionaryModel> dossierDictionaries = dictionaryCacheService.getDossierDictionary(dossierId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> { dossierDictionaries.forEach(dictionaryModel -> {
dictionaryModel.getEntries() addNewEntries(dictionaryModel, fromVersion.getDossierVersion(), newValues);
.forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
dictionaryModel.getFalsePositives()
.forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
dictionaryModel.getFalseRecommendations()
.forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
}); });
} }
return new DictionaryIncrement(newValues, version); return new DictionaryIncrement(newValues, latestVersion);
}
private void addNewEntries(DictionaryModel dictionaryModel, long versionThreshold, Set<DictionaryIncrementValue> newValues) {
dictionaryModel.getEntries()
.forEach(entry -> {
if (entry.getVersion() > versionThreshold) {
newValues.add(new DictionaryIncrementValue(entry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
dictionaryModel.getFalsePositives()
.forEach(entry -> {
if (entry.getVersion() > versionThreshold) {
newValues.add(new DictionaryIncrementValue(entry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
dictionaryModel.getFalseRecommendations()
.forEach(entry -> {
if (entry.getVersion() > versionThreshold) {
newValues.add(new DictionaryIncrementValue(entry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
} }
@ -166,125 +158,18 @@ public class DictionaryService {
try { try {
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation(); DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, currentVersion, true) : dictionaryClient.getAllTypesForDossier( List<Type> typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId,
dossierId, currentVersion,
true) : dictionaryClient.getAllTypesForDossier(dossierId,
currentVersion, currentVersion,
true); true);
if (CollectionUtils.isNotEmpty(typeResponse)) { if (CollectionUtils.isNotEmpty(typeResponse)) {
String tenantId = TenantContext.getTenantId(); String tenantId = TenantContext.getTenantId();
List<DictionaryModel> dictionary = typeResponse.stream() List<DictionaryModel> dictionary = typeResponse.stream()
.parallel() .parallel()
.map(t -> { .map(t -> mapTypeToDictionaryModel(tenantId, t, dossierTemplateId, dossierId))
TenantContext.setTenantId(tenantId);
Optional<DictionaryModel> optionalOldModel;
if (dossierId == null) {
var representation = getDossierTemplateDictionary(dossierTemplateId);
optionalOldModel = representation != null ? representation.getDictionary()
.stream()
.filter(f -> f.getType().equals(t.getType()))
.findAny() : Optional.empty();
} else {
var representation = getDossierDictionary(dossierId);
optionalOldModel = representation != null ? representation.getDictionary()
.stream()
.filter(f -> f.getType().equals(t.getType()))
.findAny() : Optional.empty();
}
Set<DictionaryEntryModel> entries = new HashSet<>();
Set<DictionaryEntryModel> falsePositives = new HashSet<>();
Set<DictionaryEntryModel> falseRecommendations = new HashSet<>();
DictionaryEntries newEntries = mapEntries(t);
var newValues = newEntries.getEntries()
.stream()
.map(DictionaryEntry::getValue)
.collect(Collectors.toSet());
var newFalsePositivesValues = newEntries.getFalsePositives()
.stream()
.map(DictionaryEntry::getValue)
.collect(Collectors.toSet());
var newFalseRecommendationsValues = newEntries.getFalseRecommendations()
.stream()
.map(DictionaryEntry::getValue)
.collect(Collectors.toSet());
optionalOldModel.ifPresent(oldDictionaryModel -> {
});
if (optionalOldModel.isPresent()) {
var oldModel = optionalOldModel.get();
if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) {
// add old entries from existing DictionaryModel but exclude lower case representation
entries.addAll(oldModel.getEntries()
.stream()
.filter(f -> !newValues.stream()
.map(s -> s.toLowerCase(Locale.ROOT))
.toList().contains(f.getValue()))
.toList());
falsePositives.addAll(oldModel.getFalsePositives()
.stream()
.filter(f -> !newFalsePositivesValues.stream()
.map(s -> s.toLowerCase(Locale.ROOT))
.toList().contains(f.getValue()))
.toList());
falseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newFalseRecommendationsValues.stream()
.map(s -> s.toLowerCase(Locale.ROOT))
.toList().contains(f.getValue()))
.toList());
} else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) {
// add old entries from existing DictionaryModel but exclude upper case representation
entries.addAll(oldModel.getEntries()
.stream()
.filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
.toList());
falsePositives.addAll(oldModel.getFalsePositives()
.stream()
.filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
.toList());
falseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
.toList());
} else {
// add old entries from existing DictionaryModel
entries.addAll(oldModel.getEntries()
.stream()
.filter(f -> !newValues.contains(f.getValue()))
.toList());
falsePositives.addAll(oldModel.getFalsePositives()
.stream()
.filter(f -> !newFalsePositivesValues.contains(f.getValue()))
.toList());
falseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newFalseRecommendationsValues.contains(f.getValue()))
.toList());
}
}
// Add Increments
entries.addAll(newEntries.getEntries());
falsePositives.addAll(newEntries.getFalsePositives());
falseRecommendations.addAll(newEntries.getFalseRecommendations());
DictionaryModel dictionaryModel = new DictionaryModel(t.getType(),
t.getRank(),
convertColor(t.getHexColor()),
t.isCaseInsensitive(),
t.isHint(),
entries,
falsePositives,
falseRecommendations,
dossierId != null);
return dictionaryModel;
})
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -301,43 +186,171 @@ public class DictionaryService {
dictionaryRepresentation.setDictionary(dictionary); dictionaryRepresentation.setDictionary(dictionary);
if (dossierId == null) { if (dossierId == null) {
addDictionaryRepresentationForDossierTemplate(dossierTemplateId, dictionaryRepresentation); dictionaryCacheService.addDictionaryRepresentationForDossierTemplate(dossierTemplateId, dictionaryRepresentation);
} else { } else {
addDictionaryRepresentationForDossier(dossierId, dictionaryRepresentation); dictionaryCacheService.addDictionaryRepresentationForDossier(dossierId, dictionaryRepresentation);
} }
} }
} catch (FeignException e) { } catch (FeignException e) {
log.warn("Got some unknown feignException", e); log.warn("Got some unknown FeignException", e);
throw e; throw e;
} }
} }
private DictionaryModel mapTypeToDictionaryModel(String tenantId, Type type, String dossierTemplateId, String dossierId) {
TenantContext.setTenantId(tenantId);
Optional<DictionaryModel> optionalOldModel = getExistingDictionaryModel(type.getType(), dossierTemplateId, dossierId);
DictionaryEntries newEntries = mapEntries(type);
Set<DictionaryEntryModel> combinedEntries = new HashSet<>(newEntries.getEntries());
Set<DictionaryEntryModel> combinedFalsePositives = new HashSet<>(newEntries.getFalsePositives());
Set<DictionaryEntryModel> combinedFalseRecommendations = new HashSet<>(newEntries.getFalseRecommendations());
optionalOldModel.ifPresent(oldModel -> handleOldEntries(oldModel, type, newEntries, combinedEntries, combinedFalsePositives, combinedFalseRecommendations));
// Add new entries
combinedEntries.addAll(newEntries.getEntries());
combinedFalsePositives.addAll(newEntries.getFalsePositives());
combinedFalseRecommendations.addAll(newEntries.getFalseRecommendations());
return new DictionaryModel(type.getType(),
type.getRank(),
convertColor(type.getHexColor()),
type.isCaseInsensitive(),
type.isHint(),
combinedEntries,
combinedFalsePositives,
combinedFalseRecommendations,
type.isDossierDictionaryOnly());
}
private Optional<DictionaryModel> getExistingDictionaryModel(String type, String dossierTemplateId, String dossierId) {
DictionaryRepresentation representation = dossierId == null ? //
dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId) : dictionaryCacheService.getDossierDictionary(dossierId);
return representation != null ? representation.getDictionary()
.stream()
.filter(f -> f.getType().equals(type))
.findAny() : Optional.empty();
}
private void handleOldEntries(DictionaryModel oldModel,
Type newType,
DictionaryEntries newEntries,
Set<DictionaryEntryModel> combinedEntries,
Set<DictionaryEntryModel> combinedFalsePositives,
Set<DictionaryEntryModel> combinedFalseRecommendations) {
if (oldModel.isCaseInsensitive() && !newType.isCaseInsensitive()) {
combinedEntries.addAll(oldModel.getEntries()
.stream()
.filter(f -> !newEntries.getEntries()
.stream()
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
.collect(Collectors.toSet()).contains(f.getValue()))
.collect(Collectors.toSet()));
combinedFalsePositives.addAll(oldModel.getFalsePositives()
.stream()
.filter(f -> !newEntries.getFalsePositives()
.stream()
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
.collect(Collectors.toSet()).contains(f.getValue()))
.collect(Collectors.toSet()));
combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newEntries.getFalseRecommendations()
.stream()
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
.collect(Collectors.toSet()).contains(f.getValue()))
.collect(Collectors.toSet()));
} else if (!oldModel.isCaseInsensitive() && newType.isCaseInsensitive()) {
combinedEntries.addAll(oldModel.getEntries()
.stream()
.filter(f -> !newEntries.getEntries()
.stream()
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
.collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT)))
.collect(Collectors.toSet()));
combinedFalsePositives.addAll(oldModel.getFalsePositives()
.stream()
.filter(f -> !newEntries.getFalsePositives()
.stream()
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
.collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT)))
.collect(Collectors.toSet()));
combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newEntries.getFalseRecommendations()
.stream()
.map(s -> s.getValue().toLowerCase(Locale.ROOT))
.collect(Collectors.toSet()).contains(f.getValue().toLowerCase(Locale.ROOT)))
.collect(Collectors.toSet()));
} else {
combinedEntries.addAll(oldModel.getEntries()
.stream()
.filter(f -> !newEntries.getEntries()
.stream()
.map(DictionaryEntryModel::getValue)
.collect(Collectors.toSet()).contains(f.getValue()))
.collect(Collectors.toSet()));
combinedFalsePositives.addAll(oldModel.getFalsePositives()
.stream()
.filter(f -> !newEntries.getFalsePositives()
.stream()
.map(DictionaryEntryModel::getValue)
.collect(Collectors.toSet()).contains(f.getValue()))
.collect(Collectors.toSet()));
combinedFalseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newEntries.getFalseRecommendations()
.stream()
.map(DictionaryEntryModel::getValue)
.collect(Collectors.toSet()).contains(f.getValue()))
.collect(Collectors.toSet()));
}
}
private DictionaryEntries mapEntries(Type type) { private DictionaryEntries mapEntries(Type type) {
Set<DictionaryEntryModel> entries = type.getEntries() != null ? new HashSet<>(type.getEntries() Set<DictionaryEntryModel> entries = type.getEntries() != null ? type.getEntries()
.stream() .stream()
.map(DictionaryEntryModel::new) .map(DictionaryEntryModel::new)
.collect(Collectors.toSet())) : new HashSet<>(); .collect(Collectors.toSet()) : new HashSet<>();
Set<DictionaryEntryModel> falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries()
Set<DictionaryEntryModel> falsePositives = type.getFalsePositiveEntries() != null ? type.getFalsePositiveEntries()
.stream() .stream()
.map(DictionaryEntryModel::new) .map(DictionaryEntryModel::new)
.collect(Collectors.toSet())) : new HashSet<>(); .collect(Collectors.toSet()) : new HashSet<>();
Set<DictionaryEntryModel> falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries()
Set<DictionaryEntryModel> falseRecommendations = type.getFalseRecommendationEntries() != null ? type.getFalseRecommendationEntries()
.stream() .stream()
.map(DictionaryEntryModel::new) .map(DictionaryEntryModel::new)
.collect(Collectors.toSet())) : new HashSet<>(); .collect(Collectors.toSet()) : new HashSet<>();
if (type.isCaseInsensitive()) { if (type.isCaseInsensitive()) {
entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
falsePositives.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); falsePositives.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
} }
log.debug("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}",
log.debug("Dictionary update returned {} entries, {} falsePositives, and {} falseRecommendations for type {}",
entries.size(), entries.size(),
falsePositives.size(), falsePositives.size(),
falseRecommendations.size(), falseRecommendations.size(),
entries); type.getType());
return new DictionaryEntries(entries, falsePositives, falseRecommendations); return new DictionaryEntries(entries, falsePositives, falseRecommendations);
} }
@ -352,138 +365,69 @@ public class DictionaryService {
@SneakyThrows @SneakyThrows
public float[] getColor(String type, String dossierTemplateId) { public float[] getColor(String type, String dossierTemplateId) {
DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap() DictionaryRepresentation dossierTemplateDictionary = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId);
DictionaryModel model = dossierTemplateDictionary.getLocalAccessMap()
.get(type); .get(type);
if (model != null) { return model != null ? model.getColor() : dossierTemplateDictionary.getDefaultColor();
return model.getColor();
}
return getDossierTemplateDictionary(dossierTemplateId).getDefaultColor();
} }
@SneakyThrows @SneakyThrows
public boolean isHint(String type, String dossierTemplateId) { public boolean isHint(String type, String dossierTemplateId) {
DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap() DictionaryModel model = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap()
.get(type); .get(type);
if (model != null) { return model != null && model.isHint();
return model.isHint();
}
return false;
}
@SneakyThrows
@Timed("redactmanager_getDeepCopyDictionary")
@Observed(name = "DictionaryService", contextualName = "deep-copy-dictionary")
public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
List<DictionaryModel> mergedDictionaries = new LinkedList<>();
DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
List<DictionaryModel> dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone()));
// Add dossier
long dossierDictionaryVersion = -1;
if (dossierDictionaryExists(dossierId)) {
DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId);
List<DictionaryModel> dossierDictionaries = dossierRepresentation.getDictionary();
dossierDictionaries.forEach(dm -> mergedDictionaries.add(dm.clone()));
return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierRepresentation.getDictionaryVersion());
} else {
return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierDictionaryVersion);
}
}
private Dictionary getDictionary(List<DictionaryModel> mergedDictionaries, DictionaryRepresentation dossierTemplateRepresentation, long dossierDictionaryVersion) {
// todo: we need caching here ? --> do not create a new one always, so we do not have as many creations of the searchImplementation
// or we add it to DictionaryRepresentation and add it here as well?
// maybe redis as well?
System.out.println("TEST ---> getDictionary");
DictionaryVersion dictionaryVersion = DictionaryVersion.builder()
.dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
.dossierVersion(dossierDictionaryVersion)
.build();
String tenantId = TenantContext.getTenantId();
Dictionary cachedDictionary = dictionaryCache.get(tenantId);
if (cachedDictionary != null) {
DictionaryVersion cachedVersion = cachedDictionary.getVersion();
boolean isUpToDate = (dictionaryVersion.getDossierTemplateVersion() <= cachedVersion.getDossierTemplateVersion())
&& (dictionaryVersion.getDossierVersion() <= cachedVersion.getDossierVersion());
if (isUpToDate) {
System.out.println("TEST ---> cached Dictionary ");
return cachedDictionary;
}
}
Dictionary dictionary = new Dictionary(mergedDictionaries.stream()
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.collect(Collectors.toList()), dictionaryVersion);
dictionaryCache.put(tenantId, dictionary);
return dictionary;
} }
@SneakyThrows @SneakyThrows
public float[] getNotRedactedColor(String dossierTemplateId) { public float[] getNotRedactedColor(String dossierTemplateId) {
return getDossierTemplateDictionary(dossierTemplateId).getNotRedactedColor(); return dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId).getNotRedactedColor();
} }
@SneakyThrows private Dictionary buildDictionary(String dossierTemplateId, String dossierId, DictionaryVersion dictionaryVersion) {
private void addDictionaryRepresentationForDossierTemplate(String dossierTemplateId, DictionaryRepresentation dictionaryRepresentation) {
tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation); List<DictionaryModel> mergedDictionaries = new LinkedList<>();
DictionaryRepresentation templateDictRep = dictionaryCacheService.getDossierTemplateDictionary(dossierTemplateId);
if (templateDictRep != null) {
templateDictRep.getDictionary()
.forEach(dm -> mergedDictionaries.add(dm.clone()));
}
if (dossierDictionaryExists(dossierId)) {
DictionaryRepresentation dossierDictRep = dictionaryCacheService.getDossierDictionary(dossierId);
if (dossierDictRep != null) {
dossierDictRep.getDictionary()
.forEach(dm -> mergedDictionaries.add(dm.clone()));
}
}
return dictionaryFactory.create(mergedDictionaries.stream()
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.collect(Collectors.toList()), dictionaryVersion);
} }
@SneakyThrows private boolean checkIfDictionaryIsUpToDate(String dossierTemplateId, String dossierId, Dictionary cachedDictionary) {
private void addDictionaryRepresentationForDossier(String dossierId, DictionaryRepresentation dictionaryRepresentation) {
tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation); long latestTemplateVersion = dictionaryClient.getVersion(dossierTemplateId);
} long latestDossierVersion = dictionaryClient.getVersionForDossier(dossierId);
DictionaryVersion cachedVersion = cachedDictionary.getVersion();
@SneakyThrows return (cachedVersion.getDossierTemplateVersion() >= latestTemplateVersion) && (cachedVersion.getDossierVersion() >= latestDossierVersion);
private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate()
.get(dossierTemplateId);
}
@SneakyThrows
private DictionaryRepresentation getDossierDictionary(String dossierId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier()
.get(dossierId);
} }
@SneakyThrows @SneakyThrows
private boolean dossierDictionaryExists(String dossierId) { private boolean dossierDictionaryExists(String dossierId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId); DictionaryRepresentation dossierDictRep = dictionaryCacheService.getDossierDictionary(dossierId);
} return dossierDictRep != null;
private Long getVersion(DictionaryRepresentation dictionaryRepresentation) {
if (dictionaryRepresentation == null) {
return null;
} else {
return dictionaryRepresentation.getDictionaryVersion();
}
} }
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -265,7 +265,7 @@ public class RedactionStorageService {
// And the cache eviction logic when a file changes after e.g. ocr is not implemented yet. // And the cache eviction logic when a file changes after e.g. ocr is not implemented yet.
// See https://knecon.atlassian.net/jira/software/c/projects/RED/boards/37?selectedIssue=RED-8106. // See https://knecon.atlassian.net/jira/software/c/projects/RED/boards/37?selectedIssue=RED-8106.
@Timed("redactmanager_getDocumentGraph") @Timed("redactmanager_getDocumentGraph")
@Cacheable(value = "documentDataCache") //@Cacheable(value = "documentDataCache")
public DocumentData getDocumentData(String dossierId, String fileId) { public DocumentData getDocumentData(String dossierId, String fileId) {
try { try {

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -56,6 +56,7 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.service.DictionaryCacheService;
import com.iqser.red.service.redaction.v1.server.service.DocumentSearchService; import com.iqser.red.service.redaction.v1.server.service.DocumentSearchService;
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService; import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService; import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService;
@ -196,6 +197,10 @@ public abstract class AbstractRedactionIntegrationTest {
@Autowired @Autowired
protected TenantMongoLiquibaseExecutor tenantMongoLiquibaseExecutor; protected TenantMongoLiquibaseExecutor tenantMongoLiquibaseExecutor;
@Autowired
DictionaryCacheService dictionaryCacheService;
protected final Map<String, List<String>> dictionary = new HashMap<>(); protected final Map<String, List<String>> dictionary = new HashMap<>();
protected final Map<String, List<String>> dossierDictionary = new HashMap<>(); protected final Map<String, List<String>> dossierDictionary = new HashMap<>();
protected final Map<String, List<String>> falsePositive = new HashMap<>(); protected final Map<String, List<String>> falsePositive = new HashMap<>();
@ -260,6 +265,7 @@ public abstract class AbstractRedactionIntegrationTest {
} }
entityLogDocumentRepository.deleteAll(); entityLogDocumentRepository.deleteAll();
entityLogEntryDocumentRepository.deleteAll(); entityLogEntryDocumentRepository.deleteAll();
dictionaryCacheService.clearAllCaches();
} }

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java
View File

@ -62,6 +62,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryFactory;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
@ -148,6 +149,8 @@ import lombok.extern.slf4j.Slf4j;
private MongoConnectionProvider mongoConnectionProvider; private MongoConnectionProvider mongoConnectionProvider;
@MockBean @MockBean
private TenantProvider tenantProvider; private TenantProvider tenantProvider;
@Autowired
private DictionaryFactory dictionaryFactory;
@Test @Test
@ -250,7 +253,7 @@ import lombok.extern.slf4j.Slf4j;
testDossierTemplate = new TestDossierTemplate(dossierTemplateToUse); testDossierTemplate = new TestDossierTemplate(dossierTemplateToUse);
when(dictionaryService.updateDictionary(any(), any())).thenReturn(new DictionaryVersion(0, 0)); when(dictionaryService.updateDictionary(any(), any())).thenReturn(new DictionaryVersion(0, 0));
when(dictionaryService.getDeepCopyDictionary(any(), any())).thenReturn(testDossierTemplate.testDictionary); when(dictionaryService.getDictionary(any(), any())).thenReturn(testDossierTemplate.testDictionary);
when(dictionaryService.getDictionaryIncrements(any(), any(), any())).thenReturn(new DictionaryIncrement(Collections.emptySet(), new DictionaryVersion(0, 0))); when(dictionaryService.getDictionaryIncrements(any(), any(), any())).thenReturn(new DictionaryIncrement(Collections.emptySet(), new DictionaryVersion(0, 0)));
when(dictionaryService.isHint(any(String.class), any())).thenAnswer(invocation -> { when(dictionaryService.isHint(any(String.class), any())).thenAnswer(invocation -> {
String type = invocation.getArgument(0); String type = invocation.getArgument(0);
@ -422,7 +425,7 @@ import lombok.extern.slf4j.Slf4j;
componentRules = new String(Files.readAllBytes(componentRuleFile.toPath())); componentRules = new String(Files.readAllBytes(componentRuleFile.toPath()));
} }
testDictionary = new Dictionary(dictionaries, new DictionaryVersion(0, 0)); testDictionary = dictionaryFactory.create(dictionaries, new DictionaryVersion(0, 0));
} }

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -252,14 +252,14 @@ public class DictionaryServiceTest {
when(dictionaryClient.getDictionaryForType("dossierType", 0L)).thenReturn(dossierType); when(dictionaryClient.getDictionaryForType("dossierType", 0L)).thenReturn(dossierType);
dictionaryService.updateDictionary("dtId", "dossierId"); dictionaryService.updateDictionary("dtId", "dossierId");
var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId"); var dict = dictionaryService.getDictionary("dtId", "dossierId");
assertThat(dict.getDictionaryModels().size()).isEqualTo(2); assertThat(dict.getDictionaryModels().size()).isEqualTo(1);
var dictModel = dict.getDictionaryModels() var dictModel = dict.getDictionaryModels()
.get(0); .get(0);
assertThat(dictModel.getType()).isEqualTo(type); assertThat(dictModel.getType()).isEqualTo(type);
assertThat(dictModel.getEntries().size()).isEqualTo(3); assertThat(dictModel.getEntries().size()).isEqualTo(3);
dictModel.getEntries() dictModel.getEntries()
.forEach(entry -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId())); .forEach(entry -> assertThat(entry.getTypeId()).isEqualTo("dossierType"));
} }
} }

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -45,10 +45,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.DictionaryCacheService;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
@ -70,6 +70,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
@Autowired @Autowired
DictionaryService dictionaryService; DictionaryService dictionaryService;
@Autowired
DictionaryCacheService dictionaryCacheService;
@Configuration @Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import(LayoutParsingServiceProcessorConfiguration.class) @Import(LayoutParsingServiceProcessorConfiguration.class)
@ -136,7 +139,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>()); dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
dictionaryService.clearTenantDictionaryCache(); dictionaryCacheService.clearAllCaches();
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf"); AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
System.out.println("Start Full integration test"); System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);

73
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionaryPerformanceTest.java
View File

@ -14,31 +14,35 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIdentifier;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch;
import com.iqser.red.service.redaction.v1.server.model.dictionary.HyperscanDictionarySearch;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.gliwka.hyperscan.wrapper.CompileErrorException;
@Disabled @Disabled
public class DictionaryPerformanceTest { public class DictionaryPerformanceTest {
private static final int LARGE_DICTIONARY_SIZE = 6_000; private static final int LARGE_DICTIONARY_SIZE = 5_000;
private static final int LARGE_TEXT_REPETITIONS = 500_000; private static final int LARGE_TEXT_REPETITIONS = 50_000;
private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " private static final String LARGE_TEXT_SAMPLE = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. "
+ "Entity_1 match text. Recommendation_1 also here. Random text continues. "; + "Entity_1 match text. Recommendation_1 also here. Random text continues. ";
@Test @Test
public void testLargeFileWithLargeDictionaries() { public void testLargeFileWithLargeDictionaries() {
Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>(); Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
// Generate dictionary values with specific terms for matching
IntStream.range(0, 6) IntStream.range(0, 6)
.forEach(i -> { .forEach(i -> {
EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION; EntityType entityType = i % 2 == 0 ? EntityType.ENTITY : EntityType.RECOMMENDATION;
boolean caseSensitive = i % 2 == 0; boolean caseSensitive = i % 2 == 0;
DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true, caseSensitive); DictionaryIdentifier identifier = new DictionaryIdentifier("Type_" + i, entityType, true,
caseSensitive);
List<String> dictionary = generateLargeDictionary(); List<String> dictionary = generateLargeDictionary();
// Add specific terms that are included in the large text for matches // Add specific terms that are included in the large text for matches
@ -51,23 +55,36 @@ public class DictionaryPerformanceTest {
dictionaryValues.put(identifier, dictionary); dictionaryValues.put(identifier, dictionary);
}); });
// Measure construction time for DictionarySearchImplementation
long dictionaryTrieConstructionStart = System.currentTimeMillis(); long dictionaryTrieConstructionStart = System.currentTimeMillis();
DictionarySearchImplementation dictionarySearchImpl = new DictionarySearchImplementation(dictionaryValues); TrieDictionarySearch dictionarySearchImpl = new TrieDictionarySearch(dictionaryValues);
long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart; long trieConstructionDuration = System.currentTimeMillis() - dictionaryTrieConstructionStart;
// Measure construction time for SearchImplementations
long searchTrieConstructionStart = System.currentTimeMillis(); long searchTrieConstructionStart = System.currentTimeMillis();
List<SearchImplementation> searchImplementations = dictionaryValues.entrySet() List<SearchImplementation> searchImplementations = dictionaryValues.entrySet().stream()
.stream() .map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive())).toList();
.map(entry -> new SearchImplementation(entry.getValue(), !entry.getKey().caseSensitive()))
.toList();
long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart; long searchTrieConstructionDuration = System.currentTimeMillis() - searchTrieConstructionStart;
// Measure construction time for HyperSearchImplementation
long hyperSearchConstructionStart = System.currentTimeMillis();
HyperscanDictionarySearch hyperSearchImpl = null;
try {
hyperSearchImpl = new HyperscanDictionarySearch(dictionaryValues);
} catch (CompileErrorException e) {
assert false : "Failed to compile HyperSearchImplementation: " + e.getMessage();
}
long hyperSearchConstructionDuration = System.currentTimeMillis() - hyperSearchConstructionStart;
String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS); String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
// Measure search time for DictionarySearchImplementation
long dictionarySearchStart = System.currentTimeMillis(); long dictionarySearchStart = System.currentTimeMillis();
List<DictionarySearchImplementation.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundaries(largeText); List<TrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl
.getBoundariesAsList(largeText);
long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart; long dictionarySearchDuration = System.currentTimeMillis() - dictionarySearchStart;
// Measure search time for SearchImplementations
long searchImplStart = System.currentTimeMillis(); long searchImplStart = System.currentTimeMillis();
List<TextRange> searchMatches = new ArrayList<>(); List<TextRange> searchMatches = new ArrayList<>();
for (SearchImplementation searchImpl : searchImplementations) { for (SearchImplementation searchImpl : searchImplementations) {
@ -75,30 +92,46 @@ public class DictionaryPerformanceTest {
} }
long searchImplDuration = System.currentTimeMillis() - searchImplStart; long searchImplDuration = System.currentTimeMillis() - searchImplStart;
// Measure search time for HyperSearchImplementation
long hyperSearchStart = System.currentTimeMillis();
List<DictionarySearch.MatchPosition> hyperMatches = hyperSearchImpl.getMatchesAsList(largeText);
long hyperSearchDuration = System.currentTimeMillis() - hyperSearchStart;
System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration); System.out.printf("Dictionary Trie construction took %d ms\n", trieConstructionDuration);
System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration, dictionaryMatches.size()); System.out.printf("DictionarySearchImplementation took %d ms and found %d matches\n", dictionarySearchDuration,
dictionaryMatches.size());
System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration); System.out.printf("Combined Trie construction took %d ms\n", searchTrieConstructionDuration);
System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration, searchMatches.size()); System.out.printf("Combined SearchImplementation took %d ms and found %d matches\n", searchImplDuration,
searchMatches.size());
System.out.printf("HyperSearchImplementation construction took %d ms\n", hyperSearchConstructionDuration);
System.out.printf("HyperSearchImplementation search took %d ms and found %d matches\n", hyperSearchDuration,
hyperMatches.size());
assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() : "Both implementations should find entities."; assert !dictionaryMatches.isEmpty() && !searchMatches.isEmpty() && !hyperMatches.isEmpty()
assertEquals("Both implementations should find the same number of matches", dictionaryMatches.size(), searchMatches.size()); : "All implementations should find entities.";
// Ensure all implementations found the same number of matches
assertEquals("DictionarySearchImplementation and SearchImplementations should find the same number of matches",
dictionaryMatches.size(), searchMatches.size());
assertEquals("DictionarySearchImplementation and HyperSearchImplementation should find the same number of matches",
dictionaryMatches.size(), hyperMatches.size());
// Close the HyperSearchImplementation to release resources
hyperSearchImpl.close();
} }
private List<String> generateLargeDictionary() { private List<String> generateLargeDictionary() {
return IntStream.range(0, LARGE_DICTIONARY_SIZE)
return IntStream.range(0, LARGE_DICTIONARY_SIZE).mapToObj(i -> RandomStringGenerator.generateRandomString()) .mapToObj(i -> RandomStringGenerator.generateRandomString())
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
static final class RandomStringGenerator { static final class RandomStringGenerator {
private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
private static final int STRING_LENGTH = 50; private static final int STRING_LENGTH = 50;
private static final SecureRandom RANDOM = new SecureRandom(); private static final SecureRandom RANDOM = new SecureRandom();
public static String generateRandomString() { public static String generateRandomString() {
StringBuilder sb = new StringBuilder(STRING_LENGTH); StringBuilder sb = new StringBuilder(STRING_LENGTH);

11
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -36,7 +36,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearchImplementation; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch;
import com.iqser.red.service.redaction.v1.server.model.dictionary.TrieDictionarySearch;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
@ -125,7 +126,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
Document document = buildGraph(filename); Document document = buildGraph(filename);
dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); Dictionary dictionary = dictionaryService.getDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
long dictionarySearchStart = System.currentTimeMillis(); long dictionarySearchStart = System.currentTimeMillis();
List<TextEntity> foundEntities = new LinkedList<>(); List<TextEntity> foundEntities = new LinkedList<>();
@ -196,7 +197,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
Document document = buildGraph(filename); Document document = buildGraph(filename);
dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); Dictionary dictionary = dictionaryService.getDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
int numberOfRuns = 1; int numberOfRuns = 1;
float totalSearchTime = 0; float totalSearchTime = 0;
@ -290,10 +291,10 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
} }
private void findEntitiesWithSearchImplementation(Document document, DictionarySearchImplementation dictionarySearchImplementation, List<TextEntity> foundEntities) { private void findEntitiesWithSearchImplementation(Document document, DictionarySearch dictionarySearch, List<TextEntity> foundEntities) {
TextBlock textBlock = document.getTextBlock(); TextBlock textBlock = document.getTextBlock();
dictionarySearchImplementation.getBoundaries(textBlock) dictionarySearch.getBoundaries(textBlock)
.filter(match -> boundaryIsSurroundedBySeparators(textBlock, match.textRange())) .filter(match -> boundaryIsSurroundedBySeparators(textBlock, match.textRange()))
.map(match -> TextEntity.initialEntityNode(match.textRange(), match.identifier().type(), match.identifier().entityType(), document)) .map(match -> TextEntity.initialEntityNode(match.textRange(), match.identifier().type(), match.identifier().entityType(), document))
.forEach(foundEntities::add); .forEach(foundEntities::add);

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -246,7 +246,7 @@ public class LiveDataIntegrationTest {
dictionaryService.updateDictionary("dossierTemplateId", "dossierId"); dictionaryService.updateDictionary("dossierTemplateId", "dossierId");
var dict = dictionaryService.getDeepCopyDictionary("dossierTemplateId", "dossierId"); var dict = dictionaryService.getDictionary("dossierTemplateId", "dossierId");
assertThat(dict.getLocalAccessMap().size()).isEqualTo(12); assertThat(dict.getLocalAccessMap().size()).isEqualTo(12);
} }