Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-10290: Improve SearchImplementation logic for dictionaries

Browse Source
This commit is contained in:
maverickstuder 2024-10-29 15:13:56 +01:00
parent f15db27a7c
commit d6739d4852
3 changed files with 88 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java
View File

@ -11,6 +11,8 @@ import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import lombok.Getter;
public class DoubleArrayTrieDictionarySearch implements DictionarySearch {
private final AhoCorasickDoubleArrayTrie<List<DictionaryIdentifierWithKeyword>> trie;
@ -41,8 +43,17 @@ public class DoubleArrayTrieDictionarySearch implements DictionarySearch {
@Override
public Stream<MatchTextRange> getBoundaries(CharSequence text) {
String lowerText = text.toString().toLowerCase(Locale.ROOT);
return getMatchTextRangeStream(text, lowerText, 0);
TextContext textContext = new TextContext(text);
return getMatchTextRangeStream(textContext);
}
@Override
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
CharSequence subText = text.subSequence(region.start(), region.end());
TextContext textContext = new TextContext(subText, region.start());
return getMatchTextRangeStream(textContext);
}
@ -53,58 +64,37 @@ public class DoubleArrayTrieDictionarySearch implements DictionarySearch {
}
@Override
public Stream<MatchTextRange> getBoundaries(CharSequence text, TextRange region) {
String subText = text.subSequence(region.start(), region.end()).toString();
String lowerSubText = subText.toLowerCase(Locale.ROOT);
return getMatchTextRangeStream(text, lowerSubText, region.start());
}
@Override
public Stream<MatchPosition> getMatches(String text) {
TextContext textContext = new TextContext(text);
List<MatchPosition> matches = new ArrayList<>();
String lowerText = text.toLowerCase(Locale.ROOT);
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(lowerText);
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(textContext.getLowerText());
for (AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit : hits) {
String matchedText = text.substring(hit.begin, hit.end);
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
MatchPosition matchPosition = new MatchPosition(idkw.identifier, hit.begin, hit.end);
if (idkw.identifier.caseSensitive()) {
if (matchedText.equals(idkw.keyword)) {
matches.add(matchPosition);
}
} else {
matches.add(matchPosition);
}
}
addMatchPositionsForHit(textContext, matches, hit);
}
return matches.stream();
}
private Stream<MatchTextRange> getMatchTextRangeStream(CharSequence text, String lowerText, int offset) {
private Stream<MatchTextRange> getMatchTextRangeStream(TextContext textContext) {
List<MatchTextRange> matches = new ArrayList<>();
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(lowerText);
List<AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>>> hits = trie.parseText(textContext.getLowerText());
for (AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit : hits) {
addMatchesForHit(text, matches, hit, offset);
addMatchesForHit(textContext, matches, hit);
}
return matches.stream();
}
private void addMatchesForHit(CharSequence text, List<MatchTextRange> matches, AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit, int offset) {
private void addMatchesForHit(TextContext textContext, List<MatchTextRange> matches, AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit) {
int start = hit.begin + offset;
int end = hit.end + offset;
String matchedText = text.subSequence(start, end).toString();
int start = textContext.getStart(hit.begin);
int end = textContext.getEnd(hit.end);
String matchedText = textContext.getMatchedText(hit.begin, hit.end);
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
@ -119,6 +109,67 @@ public class DoubleArrayTrieDictionarySearch implements DictionarySearch {
}
private void addMatchPositionsForHit(TextContext textContext, List<MatchPosition> matches, AhoCorasickDoubleArrayTrie.Hit<List<DictionaryIdentifierWithKeyword>> hit) {
int start = textContext.getStart(hit.begin);
int end = textContext.getEnd(hit.end);
String matchedText = textContext.getMatchedText(hit.begin, hit.end);
List<DictionaryIdentifierWithKeyword> idWithKeywords = hit.value;
for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) {
MatchPosition matchPosition = new MatchPosition(idkw.identifier, start, end);
if (idkw.identifier.caseSensitive()) {
if (matchedText.equals(idkw.keyword)) {
matches.add(matchPosition);
}
} else {
matches.add(matchPosition);
}
}
}
private static class TextContext {
private final CharSequence text;
@Getter
private final String lowerText;
private final int offset;
public TextContext(CharSequence text, int offset) {
this.text = text;
this.lowerText = text.toString().toLowerCase(Locale.ROOT);
this.offset = offset;
}
public TextContext(CharSequence text) {
this(text, 0);
}
public int getStart(int hitBegin) {
return hitBegin + offset;
}
public int getEnd(int hitEnd) {
return hitEnd + offset;
}
public String getMatchedText(int hitBegin, int hitEnd) {
return text.subSequence(hitBegin, hitEnd).toString();
}
}
private record DictionaryIdentifierWithKeyword(DictionaryIdentifier identifier, String keyword) {
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -31,7 +31,7 @@ public class DictionarySearchService {
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
semanticNodes.parallelStream().forEach(node -> addDictionaryEntities(dictionary, node));
semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node));
}

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java
View File

@ -10,6 +10,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie;
@ -30,7 +31,7 @@ public class DictionarySearchImplementationsTest {
@Test
//@Disabled
@Disabled
public void performanceTest() {
Map<DictionaryIdentifier, List<String>> dictionaryValues = new HashMap<>();
@ -101,9 +102,7 @@ public class DictionarySearchImplementationsTest {
System.out.printf("DoubleArrayTrieDictionarySearch search took %d ms and found %d matches\n", doubleArrayTrieSearchDuration, doubleArrayTrieMatches.size());
// Assert that all implementations found matches
assert !trieDictionaryMatches.isEmpty()
&& !searchMatches.isEmpty()
&& !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities.";
assert !trieDictionaryMatches.isEmpty() && !searchMatches.isEmpty() && !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities.";
// Ensure all implementations found the same number of matches
assertEquals(trieDictionaryMatches.size(), searchMatches.size(), "Mismatch between TrieDictionarySearch and SearchImplementations");