From d6739d4852d87a2ec6caa5f341a70329acd78ca6 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Tue, 29 Oct 2024 15:13:56 +0100 Subject: [PATCH] RED-10290: Improve SearchImplementation logic for dictionaries --- .../DoubleArrayTrieDictionarySearch.java | 117 +++++++++++++----- .../service/DictionarySearchService.java | 2 +- .../DictionarySearchImplementationsTest.java | 7 +- 3 files changed, 88 insertions(+), 38 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java index 42a2bfee..fdb42e52 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DoubleArrayTrieDictionarySearch.java @@ -11,6 +11,8 @@ import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import lombok.Getter; + public class DoubleArrayTrieDictionarySearch implements DictionarySearch { private final AhoCorasickDoubleArrayTrie> trie; @@ -41,8 +43,17 @@ public class DoubleArrayTrieDictionarySearch implements DictionarySearch { @Override public Stream getBoundaries(CharSequence text) { - String lowerText = text.toString().toLowerCase(Locale.ROOT); - return getMatchTextRangeStream(text, lowerText, 0); + TextContext textContext = new TextContext(text); + return getMatchTextRangeStream(textContext); + } + + + @Override + public Stream getBoundaries(CharSequence text, TextRange region) { + + CharSequence subText = text.subSequence(region.start(), region.end()); + TextContext textContext = new TextContext(subText, region.start()); + return getMatchTextRangeStream(textContext); } @@ -53,58 +64,37 @@ public class DoubleArrayTrieDictionarySearch implements DictionarySearch { } - @Override - public Stream getBoundaries(CharSequence text, TextRange region) { - - String subText = text.subSequence(region.start(), region.end()).toString(); - String lowerSubText = subText.toLowerCase(Locale.ROOT); - return getMatchTextRangeStream(text, lowerSubText, region.start()); - } - - @Override public Stream getMatches(String text) { + TextContext textContext = new TextContext(text); List matches = new ArrayList<>(); - String lowerText = text.toLowerCase(Locale.ROOT); - List>> hits = trie.parseText(lowerText); + List>> hits = trie.parseText(textContext.getLowerText()); for (AhoCorasickDoubleArrayTrie.Hit> hit : hits) { - String matchedText = text.substring(hit.begin, hit.end); - List idWithKeywords = hit.value; - - for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) { - MatchPosition matchPosition = new MatchPosition(idkw.identifier, hit.begin, hit.end); - if (idkw.identifier.caseSensitive()) { - if (matchedText.equals(idkw.keyword)) { - matches.add(matchPosition); - } - } else { - matches.add(matchPosition); - } - } + addMatchPositionsForHit(textContext, matches, hit); } return matches.stream(); } - private Stream getMatchTextRangeStream(CharSequence text, String lowerText, int offset) { + private Stream getMatchTextRangeStream(TextContext textContext) { List matches = new ArrayList<>(); - List>> hits = trie.parseText(lowerText); + List>> hits = trie.parseText(textContext.getLowerText()); for (AhoCorasickDoubleArrayTrie.Hit> hit : hits) { - addMatchesForHit(text, matches, hit, offset); + addMatchesForHit(textContext, matches, hit); } return matches.stream(); } - private void addMatchesForHit(CharSequence text, List matches, AhoCorasickDoubleArrayTrie.Hit> hit, int offset) { + private void addMatchesForHit(TextContext textContext, List matches, AhoCorasickDoubleArrayTrie.Hit> hit) { - int start = hit.begin + offset; - int end = hit.end + offset; - String matchedText = text.subSequence(start, end).toString(); + int start = textContext.getStart(hit.begin); + int end = textContext.getEnd(hit.end); + String matchedText = textContext.getMatchedText(hit.begin, hit.end); List idWithKeywords = hit.value; for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) { @@ -119,6 +109,67 @@ public class DoubleArrayTrieDictionarySearch implements DictionarySearch { } + private void addMatchPositionsForHit(TextContext textContext, List matches, AhoCorasickDoubleArrayTrie.Hit> hit) { + + int start = textContext.getStart(hit.begin); + int end = textContext.getEnd(hit.end); + String matchedText = textContext.getMatchedText(hit.begin, hit.end); + List idWithKeywords = hit.value; + + for (DictionaryIdentifierWithKeyword idkw : idWithKeywords) { + MatchPosition matchPosition = new MatchPosition(idkw.identifier, start, end); + if (idkw.identifier.caseSensitive()) { + if (matchedText.equals(idkw.keyword)) { + matches.add(matchPosition); + } + } else { + matches.add(matchPosition); + } + } + } + + + private static class TextContext { + + private final CharSequence text; + @Getter + private final String lowerText; + private final int offset; + + + public TextContext(CharSequence text, int offset) { + + this.text = text; + this.lowerText = text.toString().toLowerCase(Locale.ROOT); + this.offset = offset; + } + + + public TextContext(CharSequence text) { + + this(text, 0); + } + + + public int getStart(int hitBegin) { + + return hitBegin + offset; + } + + + public int getEnd(int hitEnd) { + + return hitEnd + offset; + } + + + public String getMatchedText(int hitBegin, int hitEnd) { + + return text.subSequence(hitBegin, hitEnd).toString(); + } + + } + private record DictionaryIdentifierWithKeyword(DictionaryIdentifier identifier, String keyword) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 48226c8c..fb643f4f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -31,7 +31,7 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, List semanticNodes) { - semanticNodes.parallelStream().forEach(node -> addDictionaryEntities(dictionary, node)); + semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java index f40ad83a..e028c412 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DictionarySearchImplementationsTest.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie; @@ -30,7 +31,7 @@ public class DictionarySearchImplementationsTest { @Test -//@Disabled + @Disabled public void performanceTest() { Map> dictionaryValues = new HashMap<>(); @@ -101,9 +102,7 @@ public class DictionarySearchImplementationsTest { System.out.printf("DoubleArrayTrieDictionarySearch search took %d ms and found %d matches\n", doubleArrayTrieSearchDuration, doubleArrayTrieMatches.size()); // Assert that all implementations found matches - assert !trieDictionaryMatches.isEmpty() - && !searchMatches.isEmpty() - && !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities."; + assert !trieDictionaryMatches.isEmpty() && !searchMatches.isEmpty() && !doubleArrayTrieMatches.isEmpty() : "All implementations should find entities."; // Ensure all implementations found the same number of matches assertEquals(trieDictionaryMatches.size(), searchMatches.size(), "Mismatch between TrieDictionarySearch and SearchImplementations");