Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-10290: Improve SearchImplementation logic for dictionaries

Browse Source 
* pmd fix
This commit is contained in:
maverickstuder 2024-11-07 15:03:16 +01:00
parent ab1ab0a50d
commit 8dd5517a8e
1 changed files with 27 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java
View File

@ -53,32 +53,19 @@ public class DictionarySearchImplementationsTest {
EntityType entityType = random.nextBoolean() ? EntityType.ENTITY : EntityType.RECOMMENDATION;
boolean caseSensitive = random.nextBoolean();
DictionaryIdentifier identifier = new DictionaryIdentifier(
dictionaryName,
entityType,
true,
caseSensitive
);
DictionaryIdentifier identifier = new DictionaryIdentifier(dictionaryName, entityType, true, caseSensitive);
dictionaryValues.put(identifier, dictionaryTerms);
}
// **Added dummy dictionaries as per request**
// Case-sensitive dictionary containing "Entity_1"
DictionaryIdentifier entity1Identifier = new DictionaryIdentifier(
"dummy_case_sensitive",
EntityType.ENTITY,
true,
true // Case-sensitive
DictionaryIdentifier entity1Identifier = new DictionaryIdentifier("dummy_case_sensitive", EntityType.ENTITY, true, true // Case-sensitive
);
dictionaryValues.put(entity1Identifier, List.of("Entity_1"));
// Case-insensitive dictionary containing "recommendation_1"
DictionaryIdentifier recommendation1Identifier = new DictionaryIdentifier(
"dummy_case_insensitive",
EntityType.RECOMMENDATION,
true,
false // Case-insensitive
DictionaryIdentifier recommendation1Identifier = new DictionaryIdentifier("dummy_case_insensitive", EntityType.RECOMMENDATION, true, false // Case-insensitive
);
dictionaryValues.put(recommendation1Identifier, List.of("recommendation_1"));
@ -106,7 +93,6 @@ public class DictionarySearchImplementationsTest {
DoubleArrayTrieDictionarySearch doubleArrayTrieSearchImpl = new DoubleArrayTrieDictionarySearch(keyWordToIdentifiersMap);
long doubleArrayTrieConstructionDuration = System.currentTimeMillis() - doubleArrayTrieConstructionStart;
String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
// Measure search time for TrieDictionarySearch
@ -116,7 +102,8 @@ public class DictionarySearchImplementationsTest {
// Measure search time for AnotherTrieDictionarySearch
long anotherTrieSearchStart = System.currentTimeMillis();
List<DictionarySearch.MatchTextRange> anotherTrieMatches = ahoCorasickMapDictionarySearchImpl.getBoundaries(largeText).toList();
List<DictionarySearch.MatchTextRange> anotherTrieMatches = ahoCorasickMapDictionarySearchImpl.getBoundaries(largeText)
.toList();
long anotherTrieSearchDuration = System.currentTimeMillis() - anotherTrieSearchStart;
// Measure search time for SearchImplementations
@ -152,7 +139,10 @@ public class DictionarySearchImplementationsTest {
System.out.println();
// Assert that all implementations found matches
assert !trieDictionaryMatches.isEmpty() && !anotherTrieMatches.isEmpty() && !searchMatches.isEmpty() && !doubleArrayTrieMatches.isEmpty(): "All implementations should find entities.";
assert !trieDictionaryMatches.isEmpty()
&& !anotherTrieMatches.isEmpty()
&& !searchMatches.isEmpty()
&& !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities.";
// Ensure all implementations found the same number of matches
int expectedMatches = trieDictionaryMatches.size();
@ -161,7 +151,9 @@ public class DictionarySearchImplementationsTest {
assertEquals(expectedMatches, doubleArrayTrieMatches.size(), "Mismatch between DoubleTrieDictionarySearch and DoubleArrayTrieDictionarySearch");
}
private Map<String, List<String>> loadDictionaries() {
Map<String, List<String>> dictionaries = new HashMap<>();
dictionaries.put(DICTIONARY_AUTHOR, loadDictionaryFromFile("dictionaries/CBI_author.txt"));
@ -181,11 +173,14 @@ public class DictionarySearchImplementationsTest {
return dictionaries;
}
private List<String> loadDictionaryFromFile(String filePath) {
List<String> terms = new ArrayList<>();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(
Objects.requireNonNull(getClass().getClassLoader().getResourceAsStream(filePath))))) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(Objects.requireNonNull(Thread.currentThread()
.getContextClassLoader()
.getResourceAsStream(filePath))))) {
terms = reader.lines()
.map(this::cleanDictionaryEntry)
@ -198,7 +193,9 @@ public class DictionarySearchImplementationsTest {
return terms;
}
private String cleanDictionaryEntry(String entry) {
return entry.trim();
}
@ -217,16 +214,16 @@ public class DictionarySearchImplementationsTest {
return stringToIdentifiersMap;
}
@Test
public void testMultiplePayloads() {
DoubleTrieDictionarySearch dictionarySearchImpl = new DoubleTrieDictionarySearch(Map.of(
new DictionaryIdentifier("type1", EntityType.ENTITY, false, false),
List.of("apple", "banana"),
new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false),
List.of("apple", "orange"),
new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false),
List.of("apple", "kiwi")));
DoubleTrieDictionarySearch dictionarySearchImpl = new DoubleTrieDictionarySearch(Map.of(new DictionaryIdentifier("type1", EntityType.ENTITY, false, false),
List.of("apple", "banana"),
new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false),
List.of("apple", "orange"),
new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false),
List.of("apple", "kiwi")));
List<DoubleTrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundariesAsList(
"an apple is delicious, a banana and a kiwi as well. orange is a color.");
@ -235,11 +232,12 @@ public class DictionarySearchImplementationsTest {
}
@Test
public void testDoubleArrayTrie() {
Map<String, List<String>> map = new HashMap<>();
String[] keyArray = new String[] { "hers", "his", "she", "he" };
String[] keyArray = new String[]{"hers", "his", "she", "he"};
for (String key : keyArray) {
map.put(key, List.of(key, key, key));
}