Marmelator/aho-corasick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Issue #16 Use builder pattern to create Trie

Browse Source 
Previously, there was a race condition in Trie#parseText since
it called constructFailureStates on first run without synchronization.
See https://github.com/robert-bor/aho-corasick/issues/16

This commit fixes this by using the builder pattern in order to
create a fully initialized Trie.

N.B. This changes the API
This commit is contained in:
Petter Remen 2015-07-03 12:29:31 +02:00
parent 25eeef5168
commit 9bce51e001
3 changed files with 141 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
README.md
View File

@ -51,11 +51,12 @@ Usage
----- -----
Setting up the Trie is a piece of cake: Setting up the Trie is a piece of cake:
```java ```java
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("hers"); .addKeyword("hers")
trie.addKeyword("his"); .addKeyword("his")
trie.addKeyword("she"); .addKeyword("she")
trie.addKeyword("he"); .addKeyword("he")
.build();
Collection<Emit> emits = trie.parseText("ushers"); Collection<Emit> emits = trie.parseText("ushers");
``` ```
@ -68,9 +69,11 @@ In normal situations you probably want to remove overlapping instances, retainin
matches. matches.
```java ```java
Trie trie = new Trie().removeOverlaps(); Trie trie = Trie.builder()
trie.addKeyword("hot"); .removeOverlaps()
trie.addKeyword("hot chocolate"); .addKeyword("hot")
.addKeyword("hot chocolate")
.build();
Collection<Emit> emits = trie.parseText("hot chocolate"); Collection<Emit> emits = trie.parseText("hot chocolate");
``` ```
@ -82,8 +85,10 @@ There is only one result now:
If you want the algorithm to only check for whole words, you can tell the Trie to do so: If you want the algorithm to only check for whole words, you can tell the Trie to do so:
```java ```java
Trie trie = new Trie().onlyWholeWords(); Trie trie = Trie.builder()
trie.addKeyword("sugar"); .onlyWholeWords()
.addKeyword("sugar")
.build();
Collection<Emit> emits = trie.parseText("sugarcane sugarcane sugar canesugar"); Collection<Emit> emits = trie.parseText("sugarcane sugarcane sugar canesugar");
``` ```
@ -94,8 +99,10 @@ Some text are WrItTeN in combinations of lowercase and uppercase and therefore h
the Trie to lowercase the entire searchtext to ease the matching process. the Trie to lowercase the entire searchtext to ease the matching process.
```java ```java
Trie trie = new Trie().caseInsensitive(); Trie trie = Trie.builder()
trie.addKeyword("casing"); .caseInsensitive()
.addKeyword("casing")
.build();
Collection<Emit> emits = trie.parseText("CaSiNg"); Collection<Emit> emits = trie.parseText("CaSiNg");
``` ```
@ -111,10 +118,11 @@ matches as soon as you encounter them. Let's look at an example where we want to
String speech = "The Answer to the Great Question... Of Life, " + String speech = "The Answer to the Great Question... Of Life, " +
"the Universe and Everything... Is... Forty-two,' said " + "the Universe and Everything... Is... Forty-two,' said " +
"Deep Thought, with infinite majesty and calm."; "Deep Thought, with infinite majesty and calm.";
Trie trie = new Trie().removeOverlaps().onlyWholeWords().caseInsensitive(); Trie trie = Trie.builder().removeOverlaps().onlyWholeWords().caseInsensitive()
trie.addKeyword("great question"); .addKeyword("great question")
trie.addKeyword("forty-two"); .addKeyword("forty-two")
trie.addKeyword("deep thought"); .addKeyword("deep thought")
.build();
Collection<Token> tokens = trie.tokenize(speech); Collection<Token> tokens = trie.tokenize(speech);
StringBuffer html = new StringBuffer(); StringBuffer html = new StringBuffer();
html.append("<html><body><p>"); html.append("<html><body><p>");

71
src/main/java/org/ahocorasick/trie/Trie.java
View File

@ -20,33 +20,12 @@ public class Trie {
private State rootState; private State rootState;
private boolean failureStatesConstructed = false; private Trie(TrieConfig trieConfig) {
public Trie(TrieConfig trieConfig) {
this.trieConfig = trieConfig; this.trieConfig = trieConfig;
this.rootState = new State(); this.rootState = new State();
} }
public Trie() { private void addKeyword(String keyword) {
this(new TrieConfig());
}
public Trie caseInsensitive() {
this.trieConfig.setCaseInsensitive(true);
return this;
}
public Trie removeOverlaps() {
this.trieConfig.setAllowOverlaps(false);
return this;
}
public Trie onlyWholeWords() {
this.trieConfig.setOnlyWholeWords(true);
return this;
}
public void addKeyword(String keyword) {
if (keyword == null || keyword.length() == 0) { if (keyword == null || keyword.length() == 0) {
return; return;
} }
@ -87,8 +66,6 @@ public class Trie {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Collection<Emit> parseText(String text) { public Collection<Emit> parseText(String text) {
checkForConstructedFailureStates();
int position = 0; int position = 0;
State currentState = this.rootState; State currentState = this.rootState;
List<Emit> collectedEmits = new ArrayList<Emit>(); List<Emit> collectedEmits = new ArrayList<Emit>();
@ -140,12 +117,6 @@ public class Trie {
return newCurrentState; return newCurrentState;
} }
private void checkForConstructedFailureStates() {
if (!this.failureStatesConstructed) {
constructFailureStates();
}
}
private void constructFailureStates() { private void constructFailureStates() {
Queue<State> queue = new LinkedBlockingDeque<State>(); Queue<State> queue = new LinkedBlockingDeque<State>();
@ -154,7 +125,6 @@ public class Trie {
depthOneState.setFailure(this.rootState); depthOneState.setFailure(this.rootState);
queue.add(depthOneState); queue.add(depthOneState);
} }
this.failureStatesConstructed = true;
// Second, determine the fail state for all depth > 1 state // Second, determine the fail state for all depth > 1 state
while (!queue.isEmpty()) { while (!queue.isEmpty()) {
@ -184,4 +154,41 @@ public class Trie {
} }
} }
public static TrieBuilder builder() {
return new TrieBuilder();
}
public static class TrieBuilder {
private TrieConfig trieConfig = new TrieConfig();
private Trie trie = new Trie(trieConfig);
private TrieBuilder() {}
public TrieBuilder caseInsensitive() {
this.trieConfig.setCaseInsensitive(true);
return this;
}
public TrieBuilder removeOverlaps() {
this.trieConfig.setAllowOverlaps(false);
return this;
}
public TrieBuilder onlyWholeWords() {
this.trieConfig.setOnlyWholeWords(true);
return this;
}
public TrieBuilder addKeyword(String keyword) {
trie.addKeyword(keyword);
return this;
}
public Trie build() {
trie.constructFailureStates();
return trie;
}
}
} }

140
src/test/java/org/ahocorasick/trie/TrieTest.java
View File

@ -11,8 +11,9 @@ public class TrieTest {
@Test @Test
public void keywordAndTextAreTheSame() { public void keywordAndTextAreTheSame() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("abc"); .addKeyword("abc")
.build();
Collection<Emit> emits = trie.parseText("abc"); Collection<Emit> emits = trie.parseText("abc");
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
checkEmit(iterator.next(), 0, 2, "abc"); checkEmit(iterator.next(), 0, 2, "abc");
@ -20,8 +21,9 @@ public class TrieTest {
@Test @Test
public void textIsLongerThanKeyword() { public void textIsLongerThanKeyword() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("abc"); .addKeyword("abc")
.build();
Collection<Emit> emits = trie.parseText(" abc"); Collection<Emit> emits = trie.parseText(" abc");
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
checkEmit(iterator.next(), 1, 3, "abc"); checkEmit(iterator.next(), 1, 3, "abc");
@ -29,10 +31,11 @@ public class TrieTest {
@Test @Test
public void variousKeywordsOneMatch() { public void variousKeywordsOneMatch() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("abc"); .addKeyword("abc")
trie.addKeyword("bcd"); .addKeyword("bcd")
trie.addKeyword("cde"); .addKeyword("cde")
.build();
Collection<Emit> emits = trie.parseText("bcd"); Collection<Emit> emits = trie.parseText("bcd");
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
checkEmit(iterator.next(), 0, 2, "bcd"); checkEmit(iterator.next(), 0, 2, "bcd");
@ -40,11 +43,12 @@ public class TrieTest {
@Test @Test
public void ushersTest() { public void ushersTest() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("hers"); .addKeyword("hers")
trie.addKeyword("his"); .addKeyword("his")
trie.addKeyword("she"); .addKeyword("she")
trie.addKeyword("he"); .addKeyword("he")
.build();
Collection<Emit> emits = trie.parseText("ushers"); Collection<Emit> emits = trie.parseText("ushers");
assertEquals(3, emits.size()); // she @ 3, he @ 3, hers @ 5 assertEquals(3, emits.size()); // she @ 3, he @ 3, hers @ 5
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
@ -55,8 +59,9 @@ public class TrieTest {
@Test @Test
public void misleadingTest() { public void misleadingTest() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("hers"); .addKeyword("hers")
.build();
Collection<Emit> emits = trie.parseText("h he her hers"); Collection<Emit> emits = trie.parseText("h he her hers");
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
checkEmit(iterator.next(), 9, 12, "hers"); checkEmit(iterator.next(), 9, 12, "hers");
@ -64,11 +69,12 @@ public class TrieTest {
@Test @Test
public void recipes() { public void recipes() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("veal"); .addKeyword("veal")
trie.addKeyword("cauliflower"); .addKeyword("cauliflower")
trie.addKeyword("broccoli"); .addKeyword("broccoli")
trie.addKeyword("tomatoes"); .addKeyword("tomatoes")
.build();
Collection<Emit> emits = trie.parseText("2 cauliflowers, 3 tomatoes, 4 slices of veal, 100g broccoli"); Collection<Emit> emits = trie.parseText("2 cauliflowers, 3 tomatoes, 4 slices of veal, 100g broccoli");
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
checkEmit(iterator.next(), 2, 12, "cauliflower"); checkEmit(iterator.next(), 2, 12, "cauliflower");
@ -79,9 +85,10 @@ public class TrieTest {
@Test @Test
public void longAndShortOverlappingMatch() { public void longAndShortOverlappingMatch() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("he"); .addKeyword("he")
trie.addKeyword("hehehehe"); .addKeyword("hehehehe")
.build();
Collection<Emit> emits = trie.parseText("hehehehehe"); Collection<Emit> emits = trie.parseText("hehehehehe");
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
checkEmit(iterator.next(), 0, 1, "he"); checkEmit(iterator.next(), 0, 1, "he");
@ -95,10 +102,11 @@ public class TrieTest {
@Test @Test
public void nonOverlapping() { public void nonOverlapping() {
Trie trie = new Trie().removeOverlaps(); Trie trie = Trie.builder().removeOverlaps()
trie.addKeyword("ab"); .addKeyword("ab")
trie.addKeyword("cba"); .addKeyword("cba")
trie.addKeyword("ababc"); .addKeyword("ababc")
.build();
Collection<Emit> emits = trie.parseText("ababcbab"); Collection<Emit> emits = trie.parseText("ababcbab");
assertEquals(2, emits.size()); assertEquals(2, emits.size());
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
@ -109,25 +117,27 @@ public class TrieTest {
@Test @Test
public void startOfChurchillSpeech() { public void startOfChurchillSpeech() {
Trie trie = new Trie().removeOverlaps(); Trie trie = Trie.builder().removeOverlaps()
trie.addKeyword("T"); .addKeyword("T")
trie.addKeyword("u"); .addKeyword("u")
trie.addKeyword("ur"); .addKeyword("ur")
trie.addKeyword("r"); .addKeyword("r")
trie.addKeyword("urn"); .addKeyword("urn")
trie.addKeyword("ni"); .addKeyword("ni")
trie.addKeyword("i"); .addKeyword("i")
trie.addKeyword("in"); .addKeyword("in")
trie.addKeyword("n"); .addKeyword("n")
trie.addKeyword("urning"); .addKeyword("urning")
.build();
Collection<Emit> emits = trie.parseText("Turning"); Collection<Emit> emits = trie.parseText("Turning");
assertEquals(2, emits.size()); assertEquals(2, emits.size());
} }
@Test @Test
public void partialMatch() { public void partialMatch() {
Trie trie = new Trie().onlyWholeWords(); Trie trie = Trie.builder().onlyWholeWords()
trie.addKeyword("sugar"); .addKeyword("sugar")
.build();
Collection<Emit> emits = trie.parseText("sugarcane sugarcane sugar canesugar"); // left, middle, right test Collection<Emit> emits = trie.parseText("sugarcane sugarcane sugar canesugar"); // left, middle, right test
assertEquals(1, emits.size()); // Match must not be made assertEquals(1, emits.size()); // Match must not be made
checkEmit(emits.iterator().next(), 20, 24, "sugar"); checkEmit(emits.iterator().next(), 20, 24, "sugar");
@ -135,10 +145,11 @@ public class TrieTest {
@Test @Test
public void tokenizeFullSentence() { public void tokenizeFullSentence() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("Alpha"); .addKeyword("Alpha")
trie.addKeyword("Beta"); .addKeyword("Beta")
trie.addKeyword("Gamma"); .addKeyword("Gamma")
.build();
Collection<Token> tokens = trie.tokenize("Hear: Alpha team first, Beta from the rear, Gamma in reserve"); Collection<Token> tokens = trie.tokenize("Hear: Alpha team first, Beta from the rear, Gamma in reserve");
assertEquals(7, tokens.size()); assertEquals(7, tokens.size());
Iterator<Token> tokensIt = tokens.iterator(); Iterator<Token> tokensIt = tokens.iterator();
@ -153,11 +164,12 @@ public class TrieTest {
@Test @Test
public void bug5InGithubReportedByXCurry() { public void bug5InGithubReportedByXCurry() {
Trie trie = new Trie().caseInsensitive().onlyWholeWords(); Trie trie = Trie.builder().caseInsensitive().onlyWholeWords()
trie.addKeyword("turning"); .addKeyword("turning")
trie.addKeyword("once"); .addKeyword("once")
trie.addKeyword("again"); .addKeyword("again")
trie.addKeyword("börkü"); .addKeyword("börkü")
.build();
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ"); Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
assertEquals(4, emits.size()); // Match must not be made assertEquals(4, emits.size()); // Match must not be made
Iterator<Emit> it = emits.iterator(); Iterator<Emit> it = emits.iterator();
@ -169,11 +181,12 @@ public class TrieTest {
@Test @Test
public void caseInsensitive() { public void caseInsensitive() {
Trie trie = new Trie().caseInsensitive(); Trie trie = Trie.builder().caseInsensitive()
trie.addKeyword("turning"); .addKeyword("turning")
trie.addKeyword("once"); .addKeyword("once")
trie.addKeyword("again"); .addKeyword("again")
trie.addKeyword("börkü"); .addKeyword("börkü")
.build();
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ"); Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
assertEquals(4, emits.size()); // Match must not be made assertEquals(4, emits.size()); // Match must not be made
Iterator<Emit> it = emits.iterator(); Iterator<Emit> it = emits.iterator();
@ -185,10 +198,11 @@ public class TrieTest {
@Test @Test
public void tokenizeTokensInSequence() { public void tokenizeTokensInSequence() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("Alpha"); .addKeyword("Alpha")
trie.addKeyword("Beta"); .addKeyword("Beta")
trie.addKeyword("Gamma"); .addKeyword("Gamma")
.build();
Collection<Token> tokens = trie.tokenize("Alpha Beta Gamma"); Collection<Token> tokens = trie.tokenize("Alpha Beta Gamma");
assertEquals(5, tokens.size()); assertEquals(5, tokens.size());
} }
@ -196,8 +210,9 @@ public class TrieTest {
// Test offered by XCurry, https://github.com/robert-bor/aho-corasick/issues/7 // Test offered by XCurry, https://github.com/robert-bor/aho-corasick/issues/7
@Test @Test
public void zeroLengthTestBug7InGithubReportedByXCurry() { public void zeroLengthTestBug7InGithubReportedByXCurry() {
Trie trie = new Trie().removeOverlaps().onlyWholeWords().caseInsensitive(); Trie trie = Trie.builder().removeOverlaps().onlyWholeWords().caseInsensitive()
trie.addKeyword(""); .addKeyword("")
.build();
trie.tokenize("Try a natural lip and subtle bronzer to keep all the focus on those big bright eyes with NARS Eyeshadow Duo in Rated R And the winner is... Boots No7 Advanced Renewal Anti-ageing Glycolic Peel Kit ($25 amazon.com) won most-appealing peel."); trie.tokenize("Try a natural lip and subtle bronzer to keep all the focus on those big bright eyes with NARS Eyeshadow Duo in Rated R And the winner is... Boots No7 Advanced Renewal Anti-ageing Glycolic Peel Kit ($25 amazon.com) won most-appealing peel.");
} }
@ -205,9 +220,10 @@ public class TrieTest {
@Test @Test
public void unicodeIssueBug8ReportedByDwyerk() { public void unicodeIssueBug8ReportedByDwyerk() {
String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char
Trie trie = new Trie().caseInsensitive().onlyWholeWords();
assertEquals("THIS", target.substring(5,9)); // Java does it the right way assertEquals("THIS", target.substring(5,9)); // Java does it the right way
trie.addKeyword("this"); Trie trie = Trie.builder().caseInsensitive().onlyWholeWords()
.addKeyword("this")
.build();
Collection<Emit> emits = trie.parseText(target); Collection<Emit> emits = trie.parseText(target);
assertEquals(1, emits.size()); assertEquals(1, emits.size());
Iterator<Emit> it = emits.iterator(); Iterator<Emit> it = emits.iterator();