diff --git a/src/main/java/org/ahocorasick/trie/CharacterTransition.java b/src/main/java/org/ahocorasick/trie/CharacterTransition.java index ca561ee..5b19499 100644 --- a/src/main/java/org/ahocorasick/trie/CharacterTransition.java +++ b/src/main/java/org/ahocorasick/trie/CharacterTransition.java @@ -21,10 +21,19 @@ package org.ahocorasick.trie; */ class CharacterTransition extends Transition { + /** + * Create a character transition from a position in the source string + * @param c character to match + * @param start positon of character in source string + */ public CharacterTransition(Character c, int start) { super(c, start, 1); } + /** + * Create a character transition without regard for position + * @param c character to match + */ public CharacterTransition(Character c) { this(c, 0); } diff --git a/src/main/java/org/ahocorasick/trie/Keyword.java b/src/main/java/org/ahocorasick/trie/Keyword.java index 0a6db3d..d280b0a 100644 --- a/src/main/java/org/ahocorasick/trie/Keyword.java +++ b/src/main/java/org/ahocorasick/trie/Keyword.java @@ -16,13 +16,20 @@ package org.ahocorasick.trie; /** + * Keyword encapsulates part of a potential match along with the count + * of prior source tokens consumed to create the potential match. * * @author doug.lovell */ public class Keyword implements Comparable { private final String text; private int depth; - + + /** + * Create portion of potential match + * @param text content that matches + * @param depth count of prior source tokens that comprise the match + */ public Keyword(String text, int depth) { this.text = text; this.depth = depth; @@ -37,7 +44,10 @@ public class Keyword implements Comparable { } public String toString() { - return "Keyword '" + text + "' at depth " + depth; + final String t = getText(); + final int d = getDepth(); + + return "Keyword '" + t + "' at depth " + d; } @Override diff --git a/src/main/java/org/ahocorasick/trie/State.java b/src/main/java/org/ahocorasick/trie/State.java index 215a652..fd9780f 100644 --- a/src/main/java/org/ahocorasick/trie/State.java +++ b/src/main/java/org/ahocorasick/trie/State.java @@ -27,7 +27,7 @@ import java.util.*; */ public class State { - /** effective the size of the keyword */ + /** effectively the size of the keyword */ private final int depth; /** only used for the root state to refer to itself in case no matches have been found */ @@ -54,27 +54,27 @@ public class State { this.rootState = depth == 0 ? this : null; } - private State nextState(Transition t, boolean ignoreRootState) { - State nextState = this.success.get(t); + private State nextState(Transition transition, boolean ignoreRootState) { + State nextState = this.success.get(transition); if (!ignoreRootState && nextState == null && this.rootState != null) { nextState = this.rootState; } return nextState; } - public State nextState(Transition t) { - return nextState(t, false); + public State nextState(Transition transition) { + return nextState(transition, false); } - public State nextStateIgnoreRootState(Transition t) { - return nextState(t, true); + public State nextStateIgnoreRootState(Transition transition) { + return nextState(transition, true); } - public State addState(Transition t) { - State nextState = nextStateIgnoreRootState(t); + public State addState(Transition transition) { + State nextState = nextStateIgnoreRootState(transition); if (nextState == null) { nextState = new State(this.depth+1); - this.success.put(t, nextState); + this.success.put(transition, nextState); } return nextState; } @@ -97,7 +97,7 @@ public class State { } public void addEmitString(String key) { - addEmit(new Keyword(key, depth)); + addEmit(new Keyword(key, getDepth())); } public Collection emit() { diff --git a/src/main/java/org/ahocorasick/trie/Transition.java b/src/main/java/org/ahocorasick/trie/Transition.java index f0b11b4..c4dfe9b 100644 --- a/src/main/java/org/ahocorasick/trie/Transition.java +++ b/src/main/java/org/ahocorasick/trie/Transition.java @@ -44,8 +44,10 @@ public class Transition { @Override public String toString() { - return "Transition on '" + token + "' start: " + start + - ", length: " + length; + final int s = getStart(); + final int len = getLength(); + + return "Transition on '" + token + "' start: " + s + ", length: " + len; } @Override diff --git a/src/main/java/org/ahocorasick/trie/Trie.java b/src/main/java/org/ahocorasick/trie/Trie.java index 8c72734..7c6dbab 100644 --- a/src/main/java/org/ahocorasick/trie/Trie.java +++ b/src/main/java/org/ahocorasick/trie/Trie.java @@ -81,9 +81,8 @@ public class Trie { } private class TokenStream { - private final KeywordTokenizer kwt; + private final KeywordTokenizer tokenizer; private final StringBuilder input; - private Transition lookahead; public TokenStream(CharSequence text) { input = new StringBuilder(text.length()); @@ -93,23 +92,15 @@ public class Trie { Character.toLowerCase(ch) : ch); } if (trieConfig.isOnlyWholeWords()) { - kwt = new WordTokenizer(input); + tokenizer = new WordTokenizer(input); } else { - kwt = new CharacterTokenizer(input); + tokenizer = new CharacterTokenizer(input); } - lookahead = null; } public Transition nextTransition() { - Transition next = lookahead; - if (next == null) { - next = kwt.nextTransition(); - } - else { - lookahead = null; - } - return next; + return tokenizer.nextTransition(); } public String input() { @@ -123,13 +114,13 @@ public class Trie { return; } State currentState = this.rootState; - TokenStream tknz = new TokenStream(keyword); - Transition tn = tknz.nextTransition(); - while (tn != null) { - currentState = currentState.addState(tn); - tn = tknz.nextTransition(); + TokenStream tokenStream = new TokenStream(keyword); + Transition transition = tokenStream.nextTransition(); + while (transition != null) { + currentState = currentState.addState(transition); + transition = tokenStream.nextTransition(); } - currentState.addEmitString(tknz.input()); + currentState.addEmitString(tokenStream.input()); } public Collection tokenize(String text) { @@ -167,25 +158,25 @@ public class Trie { LinkedList tknHistory = new LinkedList<>(); State currentState = this.rootState; - Transition tn = tknz.nextTransition(); - while (tn != null) { + Transition nextTransition = tknz.nextTransition(); + while (nextTransition != null) { if (flushHandler.stop()) { return; } - tknHistory.add(tn); - currentState = getState(currentState, tn, flushHandler); + tknHistory.add(nextTransition); + currentState = getState(currentState, nextTransition, flushHandler); Collection emits = currentState.emit(); int depth = currentState.getDepth(); while (depth < tknHistory.size()) { tknHistory.remove(); } + int position = nextTransition.getStart() + nextTransition.getLength(); for (Keyword emit : emits) { - int position = tn.getStart() + tn.getLength(); int start = tknHistory.get(depth - emit.getDepth()).getStart(); emitCandidateHolder.addCandidate( new Emit(start, position - 1, emit.getText())); } - tn = tknz.nextTransition(); + nextTransition = tknz.nextTransition(); } flushHandler.flush(); } diff --git a/src/main/java/org/ahocorasick/trie/WordTransition.java b/src/main/java/org/ahocorasick/trie/WordTransition.java index f12d0bc..21b5836 100644 --- a/src/main/java/org/ahocorasick/trie/WordTransition.java +++ b/src/main/java/org/ahocorasick/trie/WordTransition.java @@ -21,11 +21,20 @@ package org.ahocorasick.trie; */ public class WordTransition extends Transition { - public WordTransition(String s, int start) { - super(s, start, s.length()); + /** + * Create a transition from a position in the source string + * @param word to match + * @param start position of first character within the source string + */ + public WordTransition(String word, int start) { + super(word, start, word.length()); } - public WordTransition(String s) { - this(s, 0); + /** + * Create a transition without regard for position + * @param word to match + */ + public WordTransition(String word) { + this(word, 0); } }