Marmelator/aho-corasick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed formatting changes.

Browse Source
This commit is contained in:
ryan 2014-10-06 11:02:01 -07:00
parent df503bae43
commit a46e7dfe1d
1 changed files with 125 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
src/main/java/org/ahocorasick/trie/Trie.java
View File

@ -12,120 +12,107 @@ import java.util.concurrent.LinkedBlockingDeque;
/** /**
* *
* Based on the Aho-Corasick white paper, Bell technologies: * Based on the Aho-Corasick white paper, Bell technologies:
* ftp://163.13.200.222/assistant/bearhero/prog/%A8%E4%A5%A6/ac_bm.pdf * ftp://163.13.200.222/assistant/bearhero/prog/%A8%E4%A5%A6/ac_bm.pdf
*
* @author Robert Bor * @author Robert Bor
*/ */
public class Trie public class Trie {
{
private TrieConfig trieConfig; private TrieConfig trieConfig;
private State rootState; private State rootState;
private boolean failureStatesConstructed = false; private boolean failureStatesConstructed = false;
public Trie(TrieConfig trieConfig) public Trie(TrieConfig trieConfig) {
{ this.trieConfig = trieConfig;
this.trieConfig = trieConfig; this.rootState = new State();
this.rootState = new State(); }
}
public Trie() public Trie() {
{ this(new TrieConfig());
this(new TrieConfig()); }
}
public Trie caseInsensitive() public Trie caseInsensitive() {
{ this.trieConfig.setCaseInsensitive(true);
this.trieConfig.setCaseInsensitive(true); return this;
return this; }
}
public Trie removeOverlaps() public Trie removeOverlaps() {
{ this.trieConfig.setAllowOverlaps(false);
this.trieConfig.setAllowOverlaps(false); return this;
return this; }
}
public Trie onlyWholeWords() public Trie onlyWholeWords() {
{ this.trieConfig.setOnlyWholeWords(true);
this.trieConfig.setOnlyWholeWords(true); return this;
return this; }
}
public void addKeyword(String keyword) public void addKeyword(String keyword) {
{ if (keyword == null || keyword.length() == 0) {
if (keyword == null || keyword.length() == 0) { return;
return; }
} State currentState = this.rootState;
State currentState = this.rootState; for (Character character : keyword.toCharArray()) {
for (Character character : keyword.toCharArray()) { currentState = currentState.addState(character);
currentState = currentState.addState(character); }
} currentState.addEmit(keyword);
currentState.addEmit(keyword); }
}
public Collection<Token> tokenize(String text) public Collection<Token> tokenize(String text) {
{
Collection<Token> tokens = new ArrayList<Token>(); Collection<Token> tokens = new ArrayList<Token>();
Collection<Emit> collectedEmits = parseText(text); Collection<Emit> collectedEmits = parseText(text);
int lastCollectedPosition = -1; int lastCollectedPosition = -1;
for (Emit emit : collectedEmits) { for (Emit emit : collectedEmits) {
if (emit.getStart() - lastCollectedPosition > 1) { if (emit.getStart() - lastCollectedPosition > 1) {
tokens.add(createFragment(emit, text, lastCollectedPosition)); tokens.add(createFragment(emit, text, lastCollectedPosition));
} }
tokens.add(createMatch(emit, text)); tokens.add(createMatch(emit, text));
lastCollectedPosition = emit.getEnd(); lastCollectedPosition = emit.getEnd();
} }
if (text.length() - lastCollectedPosition > 1) { if (text.length() - lastCollectedPosition > 1) {
tokens.add(createFragment(null, text, lastCollectedPosition)); tokens.add(createFragment(null, text, lastCollectedPosition));
} }
return tokens; return tokens;
} }
private Token createFragment(Emit emit, String text, int lastCollectedPosition) private Token createFragment(Emit emit, String text, int lastCollectedPosition) {
{ return new FragmentToken(text.substring(lastCollectedPosition+1, emit == null ? text.length() : emit.getStart()));
return new FragmentToken(text.substring(lastCollectedPosition + 1, emit == null ? text.length() : emit. }
getStart()));
}
private Token createMatch(Emit emit, String text) private Token createMatch(Emit emit, String text) {
{ return new MatchToken(text.substring(emit.getStart(), emit.getEnd()+1), emit);
return new MatchToken(text.substring(emit.getStart(), emit.getEnd() + 1), emit); }
}
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Collection<Emit> parseText(String text) public Collection<Emit> parseText(String text) {
{ checkForConstructedFailureStates();
checkForConstructedFailureStates();
int position = 0; int position = 0;
State currentState = this.rootState; State currentState = this.rootState;
List<Emit> collectedEmits = new ArrayList<Emit>(); List<Emit> collectedEmits = new ArrayList<Emit>();
for (Character character : text.toCharArray()) { for (Character character : text.toCharArray()) {
if (trieConfig.isCaseInsensitive()) { if (trieConfig.isCaseInsensitive()) {
character = Character.toLowerCase(character); character = Character.toLowerCase(character);
} }
currentState = getState(currentState, character); currentState = getState(currentState, character);
storeEmits(position, currentState, collectedEmits); storeEmits(position, currentState, collectedEmits);
position++; position++;
} }
if (trieConfig.isOnlyWholeWords()) { if (trieConfig.isOnlyWholeWords()) {
removePartialMatches(text, collectedEmits); removePartialMatches(text, collectedEmits);
} }
if (!trieConfig.isAllowOverlaps()) { if (!trieConfig.isAllowOverlaps()) {
IntervalTree intervalTree = new IntervalTree((List<Intervalable>) (List<?>) collectedEmits); IntervalTree intervalTree = new IntervalTree((List<Intervalable>)(List<?>)collectedEmits);
intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits); intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
} }
return collectedEmits; return collectedEmits;
} }
public boolean matches(String text) public boolean matches(String text)
{ {
@ -135,7 +122,6 @@ public class Trie
public Emit firstMatch(String text) public Emit firstMatch(String text)
{ {
if (!trieConfig.isAllowOverlaps()) { if (!trieConfig.isAllowOverlaps()) {
// Slow path. Needs to find all the matches to detect overlaps. // Slow path. Needs to find all the matches to detect overlaps.
Collection<Emit> parseText = parseText(text); Collection<Emit> parseText = parseText(text);
@ -143,10 +129,8 @@ public class Trie
return parseText.iterator().next(); return parseText.iterator().next();
} }
} else { } else {
// Fast path. Returs first match found. // Fast path. Returs first match found.
checkForConstructedFailureStates(); checkForConstructedFailureStates();
int position = 0; int position = 0;
State currentState = this.rootState; State currentState = this.rootState;
for (Character character : text.toCharArray()) { for (Character character : text.toCharArray()) {
@ -154,12 +138,10 @@ public class Trie
character = Character.toLowerCase(character); character = Character.toLowerCase(character);
} }
currentState = getState(currentState, character); currentState = getState(currentState, character);
Collection<String> emitStrs = currentState.emit(); Collection<String> emitStrs = currentState.emit();
if (emitStrs != null && !emitStrs.isEmpty()) { if (emitStrs != null && !emitStrs.isEmpty()) {
for (String emitStr : emitStrs) { for (String emitStr : emitStrs) {
final Emit emit = new Emit(position - emitStr.length() + 1, position, emitStr); final Emit emit = new Emit(position - emitStr.length() + 1, position, emitStr);
if (trieConfig.isOnlyWholeWords()) { if (trieConfig.isOnlyWholeWords()) {
if (!isPartialMatch(text, emit)) { if (!isPartialMatch(text, emit)) {
return emit; return emit;
@ -169,12 +151,9 @@ public class Trie
} }
} }
} }
position++; position++;
} }
} }
return null; return null;
} }
@ -188,74 +167,68 @@ public class Trie
private void removePartialMatches(String searchText, List<Emit> collectedEmits) private void removePartialMatches(String searchText, List<Emit> collectedEmits)
{ {
long size = searchText.length();
List<Emit> removeEmits = new ArrayList<Emit>(); List<Emit> removeEmits = new ArrayList<Emit>();
for (Emit emit : collectedEmits) { for (Emit emit : collectedEmits) {
if (isPartialMatch(searchText, emit)) { if (isPartialMatch(searchText, emit)) {
removeEmits.add(emit); removeEmits.add(emit);
} }
} }
for (Emit removeEmit : removeEmits) { for (Emit removeEmit : removeEmits) {
collectedEmits.remove(removeEmit); collectedEmits.remove(removeEmit);
} }
} }
private State getState(State currentState, Character character) private State getState(State currentState, Character character) {
{ State newCurrentState = currentState.nextState(character);
State newCurrentState = currentState.nextState(character); while (newCurrentState == null) {
while (newCurrentState == null) { currentState = currentState.failure();
currentState = currentState.failure(); newCurrentState = currentState.nextState(character);
newCurrentState = currentState.nextState(character); }
} return newCurrentState;
return newCurrentState; }
}
private void checkForConstructedFailureStates() private void checkForConstructedFailureStates() {
{ if (!this.failureStatesConstructed) {
if (!this.failureStatesConstructed) { constructFailureStates();
constructFailureStates(); }
} }
}
private void constructFailureStates() private void constructFailureStates() {
{ Queue<State> queue = new LinkedBlockingDeque<State>();
Queue<State> queue = new LinkedBlockingDeque<State>();
// First, set the fail state of all depth 1 states to the root state // First, set the fail state of all depth 1 states to the root state
for (State depthOneState : this.rootState.getStates()) { for (State depthOneState : this.rootState.getStates()) {
depthOneState.setFailure(this.rootState); depthOneState.setFailure(this.rootState);
queue.add(depthOneState); queue.add(depthOneState);
} }
this.failureStatesConstructed = true; this.failureStatesConstructed = true;
// Second, determine the fail state for all depth > 1 state // Second, determine the fail state for all depth > 1 state
while (!queue.isEmpty()) { while (!queue.isEmpty()) {
State currentState = queue.remove(); State currentState = queue.remove();
for (Character transition : currentState.getTransitions()) { for (Character transition : currentState.getTransitions()) {
State targetState = currentState.nextState(transition); State targetState = currentState.nextState(transition);
queue.add(targetState); queue.add(targetState);
State traceFailureState = currentState.failure(); State traceFailureState = currentState.failure();
while (traceFailureState.nextState(transition) == null) { while (traceFailureState.nextState(transition) == null) {
traceFailureState = traceFailureState.failure(); traceFailureState = traceFailureState.failure();
} }
State newFailureState = traceFailureState.nextState(transition); State newFailureState = traceFailureState.nextState(transition);
targetState.setFailure(newFailureState); targetState.setFailure(newFailureState);
targetState.addEmit(newFailureState.emit()); targetState.addEmit(newFailureState.emit());
} }
} }
} }
private void storeEmits(int position, State currentState, List<Emit> collectedEmits) private void storeEmits(int position, State currentState, List<Emit> collectedEmits) {
{ Collection<String> emits = currentState.emit();
Collection<String> emits = currentState.emit(); if (emits != null && !emits.isEmpty()) {
if (emits != null && !emits.isEmpty()) { for (String emit : emits) {
for (String emit : emits) { collectedEmits.add(new Emit(position-emit.length()+1, position, emit));
collectedEmits.add(new Emit(position - emit.length() + 1, position, emit)); }
} }
} }
}
} }