Marmelator/aho-corasick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Issue #16 #20 #21 adopted pull request from remen which makes sure the failure states are constructed as part of the trie construction. This prevents the NPE which the referenced issues are complaining about.

Browse Source
This commit is contained in:
robert-bor 2015-09-22 20:14:48 +02:00
parent fcefdfdaf9
commit 023c253c93
2 changed files with 26 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/ahocorasick/trie/Trie.java
View File

@ -40,7 +40,7 @@ public class Trie {
public Collection<Token> tokenize(String text) { public Collection<Token> tokenize(String text) {
Collection<Token> tokens = new ArrayList<Token>(); Collection<Token> tokens = new ArrayList<>();
Collection<Emit> collectedEmits = parseText(text); Collection<Emit> collectedEmits = parseText(text);
int lastCollectedPosition = -1; int lastCollectedPosition = -1;
@ -86,8 +86,6 @@ public class Trie {
} }
public void parseText(CharSequence text, EmitHandler emitHandler) { public void parseText(CharSequence text, EmitHandler emitHandler) {
checkForConstructedFailureStates();
State currentState = this.rootState; State currentState = this.rootState;
for (int position = 0; position < text.length(); position++) { for (int position = 0; position < text.length(); position++) {
Character character = text.charAt(position); Character character = text.charAt(position);
@ -130,7 +128,7 @@ public class Trie {
} }
private void constructFailureStates() { private void constructFailureStates() {
Queue<State> queue = new LinkedBlockingDeque<State>(); Queue<State> queue = new LinkedBlockingDeque<>();
// First, set the fail state of all depth 1 states to the root state // First, set the fail state of all depth 1 states to the root state
for (State depthOneState : this.rootState.getStates()) { for (State depthOneState : this.rootState.getStates()) {
@ -201,6 +199,11 @@ public class Trie {
return this; return this;
} }
public TrieBuilder stopOnHit() {
trie.trieConfig.setStopOnHit(true);
return this;
}
public Trie build() { public Trie build() {
trie.constructFailureStates(); trie.constructFailureStates();
return trie; return trie;

36
src/test/java/org/ahocorasick/trie/TrieTest.java
View File

@ -46,12 +46,13 @@ public class TrieTest {
@Test @Test
public void ushersTestAndStopOnHit() { public void ushersTestAndStopOnHit() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("hers"); .addKeyword("hers")
trie.addKeyword("his"); .addKeyword("his")
trie.addKeyword("she"); .addKeyword("she")
trie.addKeyword("he"); .addKeyword("he")
trie.stopOnHit(); .stopOnHit()
.build();
Collection<Emit> emits = trie.parseText("ushers"); Collection<Emit> emits = trie.parseText("ushers");
assertEquals(2, emits.size()); // she @ 3, he @ 3, hers @ 5 assertEquals(2, emits.size()); // she @ 3, he @ 3, hers @ 5
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
@ -62,11 +63,11 @@ public class TrieTest {
@Test @Test
public void ushersTest() { public void ushersTest() {
Trie trie = Trie.builder() Trie trie = Trie.builder()
.addKeyword("hers") .addKeyword("hers")
.addKeyword("his") .addKeyword("his")
.addKeyword("she") .addKeyword("she")
.addKeyword("he") .addKeyword("he")
.build(); .build();
Collection<Emit> emits = trie.parseText("ushers"); Collection<Emit> emits = trie.parseText("ushers");
assertEquals(3, emits.size()); // she @ 3, he @ 3, hers @ 5 assertEquals(3, emits.size()); // she @ 3, he @ 3, hers @ 5
Iterator<Emit> iterator = emits.iterator(); Iterator<Emit> iterator = emits.iterator();
@ -77,11 +78,12 @@ public class TrieTest {
@Test @Test
public void ushersTestByCallback() { public void ushersTestByCallback() {
Trie trie = new Trie(); Trie trie = Trie.builder()
trie.addKeyword("hers"); .addKeyword("hers")
trie.addKeyword("his"); .addKeyword("his")
trie.addKeyword("she"); .addKeyword("she")
trie.addKeyword("he"); .addKeyword("he")
.build();
final List<Emit> emits = new ArrayList<>(); final List<Emit> emits = new ArrayList<>();
EmitHandler emitHandler = new EmitHandler() { EmitHandler emitHandler = new EmitHandler() {
@ -262,7 +264,7 @@ public class TrieTest {
@Test @Test
public void unicodeIssueBug8ReportedByDwyerk() { public void unicodeIssueBug8ReportedByDwyerk() {
String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char
assertEquals("THIS", target.substring(5,9)); // Java does it the right way assertEquals("THIS", target.substring(5, 9)); // Java does it the right way
Trie trie = Trie.builder().caseInsensitive().onlyWholeWords() Trie trie = Trie.builder().caseInsensitive().onlyWholeWords()
.addKeyword("this") .addKeyword("this")
.build(); .build();