Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-10290: Improve SearchImplementation logic for dictionaries

Browse Source 
* pmd fix
This commit is contained in:
maverickstuder 2024-11-07 15:03:16 +01:00
parent ab1ab0a50d
commit 8dd5517a8e
1 changed files with 27 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java
View File

@ -53,32 +53,19 @@ public class DictionarySearchImplementationsTest {
EntityType entityType = random.nextBoolean() ? EntityType.ENTITY : EntityType.RECOMMENDATION; EntityType entityType = random.nextBoolean() ? EntityType.ENTITY : EntityType.RECOMMENDATION;
boolean caseSensitive = random.nextBoolean(); boolean caseSensitive = random.nextBoolean();
DictionaryIdentifier identifier = new DictionaryIdentifier( DictionaryIdentifier identifier = new DictionaryIdentifier(dictionaryName, entityType, true, caseSensitive);
dictionaryName,
entityType,
true,
caseSensitive
);
dictionaryValues.put(identifier, dictionaryTerms); dictionaryValues.put(identifier, dictionaryTerms);
} }
// **Added dummy dictionaries as per request** // **Added dummy dictionaries as per request**
// Case-sensitive dictionary containing "Entity_1" // Case-sensitive dictionary containing "Entity_1"
DictionaryIdentifier entity1Identifier = new DictionaryIdentifier( DictionaryIdentifier entity1Identifier = new DictionaryIdentifier("dummy_case_sensitive", EntityType.ENTITY, true, true // Case-sensitive
"dummy_case_sensitive",
EntityType.ENTITY,
true,
true // Case-sensitive
); );
dictionaryValues.put(entity1Identifier, List.of("Entity_1")); dictionaryValues.put(entity1Identifier, List.of("Entity_1"));
// Case-insensitive dictionary containing "recommendation_1" // Case-insensitive dictionary containing "recommendation_1"
DictionaryIdentifier recommendation1Identifier = new DictionaryIdentifier( DictionaryIdentifier recommendation1Identifier = new DictionaryIdentifier("dummy_case_insensitive", EntityType.RECOMMENDATION, true, false // Case-insensitive
"dummy_case_insensitive",
EntityType.RECOMMENDATION,
true,
false // Case-insensitive
); );
dictionaryValues.put(recommendation1Identifier, List.of("recommendation_1")); dictionaryValues.put(recommendation1Identifier, List.of("recommendation_1"));
@ -106,7 +93,6 @@ public class DictionarySearchImplementationsTest {
DoubleArrayTrieDictionarySearch doubleArrayTrieSearchImpl = new DoubleArrayTrieDictionarySearch(keyWordToIdentifiersMap); DoubleArrayTrieDictionarySearch doubleArrayTrieSearchImpl = new DoubleArrayTrieDictionarySearch(keyWordToIdentifiersMap);
long doubleArrayTrieConstructionDuration = System.currentTimeMillis() - doubleArrayTrieConstructionStart; long doubleArrayTrieConstructionDuration = System.currentTimeMillis() - doubleArrayTrieConstructionStart;
String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS); String largeText = LARGE_TEXT_SAMPLE.repeat(LARGE_TEXT_REPETITIONS);
// Measure search time for TrieDictionarySearch // Measure search time for TrieDictionarySearch
@ -116,7 +102,8 @@ public class DictionarySearchImplementationsTest {
// Measure search time for AnotherTrieDictionarySearch // Measure search time for AnotherTrieDictionarySearch
long anotherTrieSearchStart = System.currentTimeMillis(); long anotherTrieSearchStart = System.currentTimeMillis();
List<DictionarySearch.MatchTextRange> anotherTrieMatches = ahoCorasickMapDictionarySearchImpl.getBoundaries(largeText).toList(); List<DictionarySearch.MatchTextRange> anotherTrieMatches = ahoCorasickMapDictionarySearchImpl.getBoundaries(largeText)
.toList();
long anotherTrieSearchDuration = System.currentTimeMillis() - anotherTrieSearchStart; long anotherTrieSearchDuration = System.currentTimeMillis() - anotherTrieSearchStart;
// Measure search time for SearchImplementations // Measure search time for SearchImplementations
@ -152,7 +139,10 @@ public class DictionarySearchImplementationsTest {
System.out.println(); System.out.println();
// Assert that all implementations found matches // Assert that all implementations found matches
assert !trieDictionaryMatches.isEmpty() && !anotherTrieMatches.isEmpty() && !searchMatches.isEmpty() && !doubleArrayTrieMatches.isEmpty(): "All implementations should find entities."; assert !trieDictionaryMatches.isEmpty()
&& !anotherTrieMatches.isEmpty()
&& !searchMatches.isEmpty()
&& !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities.";
// Ensure all implementations found the same number of matches // Ensure all implementations found the same number of matches
int expectedMatches = trieDictionaryMatches.size(); int expectedMatches = trieDictionaryMatches.size();
@ -161,7 +151,9 @@ public class DictionarySearchImplementationsTest {
assertEquals(expectedMatches, doubleArrayTrieMatches.size(), "Mismatch between DoubleTrieDictionarySearch and DoubleArrayTrieDictionarySearch"); assertEquals(expectedMatches, doubleArrayTrieMatches.size(), "Mismatch between DoubleTrieDictionarySearch and DoubleArrayTrieDictionarySearch");
} }
private Map<String, List<String>> loadDictionaries() { private Map<String, List<String>> loadDictionaries() {
Map<String, List<String>> dictionaries = new HashMap<>(); Map<String, List<String>> dictionaries = new HashMap<>();
dictionaries.put(DICTIONARY_AUTHOR, loadDictionaryFromFile("dictionaries/CBI_author.txt")); dictionaries.put(DICTIONARY_AUTHOR, loadDictionaryFromFile("dictionaries/CBI_author.txt"));
@ -181,11 +173,14 @@ public class DictionarySearchImplementationsTest {
return dictionaries; return dictionaries;
} }
private List<String> loadDictionaryFromFile(String filePath) { private List<String> loadDictionaryFromFile(String filePath) {
List<String> terms = new ArrayList<>(); List<String> terms = new ArrayList<>();
try (BufferedReader reader = new BufferedReader(new InputStreamReader( try (BufferedReader reader = new BufferedReader(new InputStreamReader(Objects.requireNonNull(Thread.currentThread()
Objects.requireNonNull(getClass().getClassLoader().getResourceAsStream(filePath))))) { .getContextClassLoader()
.getResourceAsStream(filePath))))) {
terms = reader.lines() terms = reader.lines()
.map(this::cleanDictionaryEntry) .map(this::cleanDictionaryEntry)
@ -198,7 +193,9 @@ public class DictionarySearchImplementationsTest {
return terms; return terms;
} }
private String cleanDictionaryEntry(String entry) { private String cleanDictionaryEntry(String entry) {
return entry.trim(); return entry.trim();
} }
@ -217,16 +214,16 @@ public class DictionarySearchImplementationsTest {
return stringToIdentifiersMap; return stringToIdentifiersMap;
} }
@Test @Test
public void testMultiplePayloads() { public void testMultiplePayloads() {
DoubleTrieDictionarySearch dictionarySearchImpl = new DoubleTrieDictionarySearch(Map.of( DoubleTrieDictionarySearch dictionarySearchImpl = new DoubleTrieDictionarySearch(Map.of(new DictionaryIdentifier("type1", EntityType.ENTITY, false, false),
new DictionaryIdentifier("type1", EntityType.ENTITY, false, false), List.of("apple", "banana"),
List.of("apple", "banana"), new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false),
new DictionaryIdentifier("type2", EntityType.RECOMMENDATION, false, false), List.of("apple", "orange"),
List.of("apple", "orange"), new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false),
new DictionaryIdentifier("type3", EntityType.FALSE_POSITIVE, false, false), List.of("apple", "kiwi")));
List.of("apple", "kiwi")));
List<DoubleTrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundariesAsList( List<DoubleTrieDictionarySearch.MatchTextRange> dictionaryMatches = dictionarySearchImpl.getBoundariesAsList(
"an apple is delicious, a banana and a kiwi as well. orange is a color."); "an apple is delicious, a banana and a kiwi as well. orange is a color.");
@ -235,11 +232,12 @@ public class DictionarySearchImplementationsTest {
} }
@Test @Test
public void testDoubleArrayTrie() { public void testDoubleArrayTrie() {
Map<String, List<String>> map = new HashMap<>(); Map<String, List<String>> map = new HashMap<>();
String[] keyArray = new String[] { "hers", "his", "she", "he" }; String[] keyArray = new String[]{"hers", "his", "she", "he"};
for (String key : keyArray) { for (String key : keyArray) {
map.put(key, List.of(key, key, key)); map.put(key, List.of(key, key, key));
} }