Marmelator/aho-corasick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added missing override annotations. Added final modifier to Interval member variables. Updated documentation for ignoreCase (issue #33) and moved the ignore methods to the top of the builder to reflect their preferred calling order.

Browse Source
This commit is contained in:
djarvis 2016-11-29 22:34:55 -08:00 committed by robert-bor
parent 89e7efab72
commit 5edf6d8126
2 changed files with 158 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/org/ahocorasick/interval/Interval.java
View File

@ -2,8 +2,8 @@ package org.ahocorasick.interval;
public class Interval implements Intervalable { public class Interval implements Intervalable {
private int start; private final int start;
private int end; private final int end;
/** /**
* Constructs an interval with a start and end position. * Constructs an interval with a start and end position.
@ -21,6 +21,7 @@ public class Interval implements Intervalable {
* *
* @return A number between 0 (start of text) and the text length. * @return A number between 0 (start of text) and the text length.
*/ */
@Override
public int getStart() { public int getStart() {
return this.start; return this.start;
} }
@ -30,6 +31,7 @@ public class Interval implements Intervalable {
* *
* @return A number between getStart() + 1 and the text length. * @return A number between getStart() + 1 and the text length.
*/ */
@Override
public int getEnd() { public int getEnd() {
return this.end; return this.end;
} }
@ -39,6 +41,7 @@ public class Interval implements Intervalable {
* *
* @return The end position less the start position, plus one. * @return The end position less the start position, plus one.
*/ */
@Override
public int size() { public int size() {
return end - start + 1; return end - start + 1;
} }
@ -47,6 +50,7 @@ public class Interval implements Intervalable {
* Answers whether the given interval overlaps this interval * Answers whether the given interval overlaps this interval
* instance. * instance.
* *
* @param other
* @return true The intervals overlap. * @return true The intervals overlap.
*/ */
public boolean overlapsWith(final Interval other) { public boolean overlapsWith(final Interval other) {

296
src/main/java/org/ahocorasick/trie/Trie.java
View File

@ -1,24 +1,20 @@
package org.ahocorasick.trie; package org.ahocorasick.trie;
import org.ahocorasick.interval.IntervalTree; import static java.lang.Character.isWhitespace;
import org.ahocorasick.interval.Intervalable;
import org.ahocorasick.trie.handler.DefaultEmitHandler;
import org.ahocorasick.trie.handler.EmitHandler;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Queue; import java.util.Queue;
import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.LinkedBlockingDeque;
import org.ahocorasick.interval.IntervalTree;
import static java.lang.Character.*; import org.ahocorasick.interval.Intervalable;
import org.ahocorasick.trie.handler.DefaultEmitHandler;
import java.lang.Character; import org.ahocorasick.trie.handler.EmitHandler;
/** /**
* Based on the Aho-Corasick white paper, Bell technologies: * Based on the Aho-Corasick white paper, Bell technologies:
* http://cr.yp.to/bib/1975/aho.pdf * http://cr.yp.to/bib/1975/aho.pdf
* *
* @author Robert Bor * @author Robert Bor
*/ */
public class Trie { public class Trie {
@ -31,41 +27,66 @@ public class Trie {
this.trieConfig = trieConfig; this.trieConfig = trieConfig;
this.rootState = new State(); this.rootState = new State();
} }
/** /**
* Used by the builder to add a text search keyword. * Used by the builder to add a text search keyword.
* *
* @param keyword The search term to add to the list of search terms. * @param keyword The search term to add to the list of search terms.
*
* @throws NullPointerException if the keyword is null. * @throws NullPointerException if the keyword is null.
*/ */
private void addKeyword(String keyword) { private void addKeyword(String keyword) {
if (keyword.length() > 0) { if( keyword.isEmpty() ) {
if (isCaseInsensitive()) { return;
keyword = keyword.toLowerCase();
}
addState(keyword).addEmit(keyword);
} }
if( isCaseInsensitive() ) {
keyword = keyword.toLowerCase();
}
addState(keyword).addEmit(keyword);
}
/**
* Delegates to addKeyword.
*
* @param keywords List of search term to add to the list of search terms.
*/
private void addKeywords( final String[] keywords ) {
for( final String keyword : keywords ) {
addKeyword( keyword );
}
}
/**
* Delegates to addKeyword.
*
* @param keywords List of search term to add to the list of search terms.
*/
private void addKeywords( final Collection<String> keywords ) {
for( final String keyword : keywords ) {
addKeyword( keyword );
}
} }
private State addState(final String keyword) { private State addState(final String keyword) {
return getRootState().addState(keyword); return getRootState().addState(keyword);
} }
public Collection<Token> tokenize(final String text) { public Collection<Token> tokenize(final String text) {
final Collection<Token> tokens = new ArrayList<>(); final Collection<Token> tokens = new ArrayList<>();
final Collection<Emit> collectedEmits = parseText(text); final Collection<Emit> collectedEmits = parseText(text);
int lastCollectedPosition = -1; int lastCollectedPosition = -1;
for (final Emit emit : collectedEmits) { for (final Emit emit : collectedEmits) {
if (emit.getStart() - lastCollectedPosition > 1) { if (emit.getStart() - lastCollectedPosition > 1) {
tokens.add(createFragment(emit, text, lastCollectedPosition)); tokens.add(createFragment(emit, text, lastCollectedPosition));
} }
tokens.add(createMatch(emit, text)); tokens.add(createMatch(emit, text));
lastCollectedPosition = emit.getEnd(); lastCollectedPosition = emit.getEnd();
} }
if (text.length() - lastCollectedPosition > 1) { if (text.length() - lastCollectedPosition > 1) {
tokens.add(createFragment(null, text, lastCollectedPosition)); tokens.add(createFragment(null, text, lastCollectedPosition));
} }
@ -73,15 +94,12 @@ public class Trie {
return tokens; return tokens;
} }
private Token createFragment( private Token createFragment(final Emit emit, final String text, final int lastCollectedPosition) {
final Emit emit, return new FragmentToken(text.substring(lastCollectedPosition+1, emit == null ? text.length() : emit.getStart()));
final String text,
final int lastCollectedPosition) {
return new FragmentToken(text.substring(lastCollectedPosition + 1, emit == null ? text.length() : emit.getStart()));
} }
private Token createMatch(final Emit emit, final String text) { private Token createMatch(Emit emit, String text) {
return new MatchToken(text.substring(emit.getStart(), emit.getEnd() + 1), emit); return new MatchToken(text.substring(emit.getStart(), emit.getEnd()+1), emit);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -100,7 +118,7 @@ public class Trie {
} }
if (!trieConfig.isAllowOverlaps()) { if (!trieConfig.isAllowOverlaps()) {
IntervalTree intervalTree = new IntervalTree((List<Intervalable>) (List<?>) collectedEmits); IntervalTree intervalTree = new IntervalTree((List<Intervalable>)(List<?>)collectedEmits);
intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits); intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
} }
@ -113,15 +131,15 @@ public class Trie {
public void parseText(final CharSequence text, final EmitHandler emitHandler) { public void parseText(final CharSequence text, final EmitHandler emitHandler) {
State currentState = getRootState(); State currentState = getRootState();
for (int position = 0; position < text.length(); position++) { for (int position = 0; position < text.length(); position++) {
Character character = text.charAt(position); Character character = text.charAt(position);
// TODO: Maybe lowercase the entire string at once? // TODO: Maybe lowercase the entire string at once?
if (trieConfig.isCaseInsensitive()) { if (trieConfig.isCaseInsensitive()) {
character = toLowerCase(character); character = Character.toLowerCase(character);
} }
currentState = getState(currentState, character); currentState = getState(currentState, character);
if (storeEmits(position, currentState, emitHandler) && trieConfig.isStopOnHit()) { if (storeEmits(position, currentState, emitHandler) && trieConfig.isStopOnHit()) {
return; return;
@ -129,28 +147,35 @@ public class Trie {
} }
} }
/**
* The first matching text sequence.
*
* @param text The text to search for keywords.
* @return null if no matches found.
*/
public Emit firstMatch(final CharSequence text) { public Emit firstMatch(final CharSequence text) {
if (!trieConfig.isAllowOverlaps()) { if (!trieConfig.isAllowOverlaps()) {
// Slow path. Needs to find all the matches to detect overlaps. // Slow path. Needs to find all the matches to detect overlaps.
Collection<Emit> parseText = parseText(text); final Collection<Emit> parseText = parseText(text);
if (parseText != null && !parseText.isEmpty()) { if (parseText != null && !parseText.isEmpty()) {
return parseText.iterator().next(); return parseText.iterator().next();
} }
} else { } else {
// Fast path. Returns first match found. // Fast path. Returns first match found.
State currentState = getRootState(); State currentState = getRootState();
for (int position = 0; position < text.length(); position++) { for (int position = 0; position < text.length(); position++) {
Character character = text.charAt(position); Character character = text.charAt(position);
// TODO: Lowercase the entire string at once? // TODO: Lowercase the entire string at once?
if (trieConfig.isCaseInsensitive()) { if (trieConfig.isCaseInsensitive()) {
character = toLowerCase(character); character = Character.toLowerCase(character);
} }
currentState = getState(currentState, character); currentState = getState(currentState, character);
Collection<String> emitStrs = currentState.emit(); Collection<String> emitStrs = currentState.emit();
if (emitStrs != null && !emitStrs.isEmpty()) { if (emitStrs != null && !emitStrs.isEmpty()) {
for (final String emitStr : emitStrs) { for (final String emitStr : emitStrs) {
final Emit emit = new Emit(position - emitStr.length() + 1, position, emitStr); final Emit emit = new Emit(position - emitStr.length() + 1, position, emitStr);
@ -165,26 +190,26 @@ public class Trie {
} }
} }
} }
return null; return null;
} }
private boolean isPartialMatch(final CharSequence searchText, final Emit emit) { private boolean isPartialMatch(final CharSequence searchText, final Emit emit) {
return (emit.getStart() != 0 && return (emit.getStart() != 0 &&
isAlphabetic(searchText.charAt(emit.getStart() - 1))) || Character.isAlphabetic(searchText.charAt(emit.getStart() - 1))) ||
(emit.getEnd() + 1 != searchText.length() && (emit.getEnd() + 1 != searchText.length() &&
isAlphabetic(searchText.charAt(emit.getEnd() + 1))); Character.isAlphabetic(searchText.charAt(emit.getEnd() + 1)));
} }
private void removePartialMatches(final CharSequence searchText, final List<Emit> collectedEmits) { private void removePartialMatches(final CharSequence searchText, final List<Emit> collectedEmits) {
final List<Emit> removeEmits = new ArrayList<>(); final List<Emit> removeEmits = new ArrayList<>();
for (final Emit emit : collectedEmits) { for (final Emit emit : collectedEmits) {
if (isPartialMatch(searchText, emit)) { if (isPartialMatch(searchText, emit)) {
removeEmits.add(emit); removeEmits.add(emit);
} }
} }
for (final Emit removeEmit : removeEmits) { for (final Emit removeEmit : removeEmits) {
collectedEmits.remove(removeEmit); collectedEmits.remove(removeEmit);
} }
@ -193,30 +218,29 @@ public class Trie {
private void removePartialMatchesWhiteSpaceSeparated(final CharSequence searchText, final List<Emit> collectedEmits) { private void removePartialMatchesWhiteSpaceSeparated(final CharSequence searchText, final List<Emit> collectedEmits) {
final long size = searchText.length(); final long size = searchText.length();
final List<Emit> removeEmits = new ArrayList<>(); final List<Emit> removeEmits = new ArrayList<>();
for (final Emit emit : collectedEmits) { for (final Emit emit : collectedEmits) {
if ((emit.getStart() == 0 || isWhitespace(searchText.charAt(emit.getStart() - 1))) && if ((emit.getStart() == 0 || isWhitespace(searchText.charAt(emit.getStart() - 1))) &&
(emit.getEnd() + 1 == size || isWhitespace(searchText.charAt(emit.getEnd() + 1)))) { (emit.getEnd() + 1 == size || isWhitespace(searchText.charAt(emit.getEnd() + 1)))) {
continue; continue;
} }
removeEmits.add(emit); removeEmits.add(emit);
} }
for (final Emit removeEmit : removeEmits) { for (final Emit removeEmit : removeEmits) {
collectedEmits.remove(removeEmit); collectedEmits.remove(removeEmit);
} }
} }
private State getState(final State initialState, final Character character) { private State getState(State currentState, final Character character) {
State currentState = initialState; State newCurrentState = currentState.nextState(character);
State updatedState = currentState.nextState(character);
while (newCurrentState == null) {
while (updatedState == null) {
currentState = currentState.failure(); currentState = currentState.failure();
updatedState = currentState.nextState(character); newCurrentState = currentState.nextState(character);
} }
return updatedState; return newCurrentState;
} }
private void constructFailureStates() { private void constructFailureStates() {
@ -249,13 +273,10 @@ public class Trie {
} }
} }
private boolean storeEmits( private boolean storeEmits(final int position, final State currentState, final EmitHandler emitHandler) {
final int position,
final State currentState,
final EmitHandler emitHandler) {
boolean emitted = false; boolean emitted = false;
final Collection<String> emits = currentState.emit(); final Collection<String> emits = currentState.emit();
// TODO: The check for empty might be superfluous. // TODO: The check for empty might be superfluous.
if (emits != null && !emits.isEmpty()) { if (emits != null && !emits.isEmpty()) {
for (final String emit : emits) { for (final String emit : emits) {
@ -263,31 +284,27 @@ public class Trie {
emitted = true; emitted = true;
} }
} }
return emitted; return emitted;
} }
private boolean isCaseInsensitive() { private boolean isCaseInsensitive() {
return trieConfig.isCaseInsensitive(); return trieConfig.isCaseInsensitive();
} }
private State getRootState() { private State getRootState() {
return this.rootState; return this.rootState;
} }
/** /**
* Constructs a TrieBuilder instance for configuring the Trie using a fluent * Provides a fluent interface for constructing Trie instances.
* interface. *
*
* @return The builder used to configure its Trie. * @return The builder used to configure its Trie.
*/ */
public static TrieBuilder builder() { public static TrieBuilder builder() {
return new TrieBuilder(); return new TrieBuilder();
} }
/**
* Provides a fluent interface for constructing Trie instances.
*/
public static class TrieBuilder { public static class TrieBuilder {
private final TrieConfig trieConfig = new TrieConfig(); private final TrieConfig trieConfig = new TrieConfig();
@ -297,73 +314,75 @@ public class Trie {
/** /**
* Default (empty) constructor. * Default (empty) constructor.
*/ */
private TrieBuilder() { private TrieBuilder() {}
}
/** /**
* Adds a keyword to the Trie's list of text search keywords. * Configure the Trie to ignore case when searching for keywords in
* * the text. This must be called before calling addKeyword because
* @param keyword The keyword to add to the list. * the algorithm converts keywords to lowercase as they are added,
* @return This builder. * depending on this case sensitivity setting.
* @throws NullPointerException if the keyword is null. *
*/
public TrieBuilder addKeyword(final CharSequence keyword) {
getTrie().addKeyword(keyword.toString());
return this;
}
/**
* Adds a list of keywords to the Trie's list of text search keywords.
*
* @param keywords The keywords to add to the list.
* @return This builder.
*/
public TrieBuilder addKeywords(final CharSequence... keywords) {
for (final CharSequence keyword : keywords) {
addKeyword(keyword);
}
return this;
}
/**
* Adds a list of keywords to the Trie's list of text search keywords.
*
* @param keywords The keywords to add to the list.
* @return This builder.
*/
public TrieBuilder addKeywords(final Collection<CharSequence> keywords) {
return addKeywords(keywords.toArray(new CharSequence[keywords.size()]));
}
/**
* Configure the Trie to ignore case when searching for keywords in the
* text.
*
* @return This builder. * @return This builder.
*/ */
public TrieBuilder ignoreCase() { public TrieBuilder ignoreCase() {
getTrieConfig().setCaseInsensitive(true); this.trieConfig.setCaseInsensitive(true);
return this; return this;
} }
/** /**
* Configure the Trie to ignore overlapping keywords. * Configure the Trie to ignore overlapping keywords.
* *
* @return This builder. * @return This builder.
*/ */
public TrieBuilder ignoreOverlaps() { public TrieBuilder ignoreOverlaps() {
getTrieConfig().setAllowOverlaps(false); this.trieConfig.setAllowOverlaps(false);
return this; return this;
} }
/**
* Adds a keyword to the Trie's list of text search keywords.
*
* @param keyword The keyword to add to the list.
*
* @return This builder.
* @throws NullPointerException if the keyword is null.
*/
public TrieBuilder addKeyword(final String keyword) {
this.trie.addKeyword(keyword);
return this;
}
/**
* Adds a list of keywords to the Trie's list of text search keywords.
*
* @param keywords The keywords to add to the list.
*
* @return This builder.
*/
public TrieBuilder addKeywords(final String... keywords) {
this.trie.addKeywords(keywords);
return this;
}
/**
* Adds a list of keywords to the Trie's list of text search keywords.
*
* @param keywords The keywords to add to the list.
*
* @return This builder.
*/
public TrieBuilder addKeywords(final Collection<String> keywords) {
this.trie.addKeywords(keywords);
return this;
}
/** /**
* Configure the Trie to match whole keywords in the text. * Configure the Trie to match whole keywords in the text.
* *
* @return This builder. * @return This builder.
*/ */
public TrieBuilder onlyWholeWords() { public TrieBuilder onlyWholeWords() {
getTrieConfig().setOnlyWholeWords(true); this.trieConfig.setOnlyWholeWords(true);
return this; return this;
} }
@ -371,52 +390,39 @@ public class Trie {
* Configure the Trie to match whole keywords that are separated by * Configure the Trie to match whole keywords that are separated by
* whitespace in the text. For example, "this keyword thatkeyword" * whitespace in the text. For example, "this keyword thatkeyword"
* would only match the first occurrence of "keyword". * would only match the first occurrence of "keyword".
* *
* @return This builder. * @return This builder.
*/ */
public TrieBuilder onlyWholeWordsWhiteSpaceSeparated() { public TrieBuilder onlyWholeWordsWhiteSpaceSeparated() {
getTrieConfig().setOnlyWholeWordsWhiteSpaceSeparated(true); this.trieConfig.setOnlyWholeWordsWhiteSpaceSeparated(true);
return this; return this;
} }
/** /**
* Configure the Trie to stop searching for matches after the first * Configure the Trie to stop after the first keyword is found in the
* keyword is found in the text. * text.
* *
* @return This builder. * @return This builder.
*/ */
public TrieBuilder onlyFirstMatch() { public TrieBuilder stopOnHit() {
getTrieConfig().setStopOnHit(true); trie.trieConfig.setStopOnHit(true);
return this; return this;
} }
/** /**
* Construct the Trie using the builder settings. * Configure the Trie based on the builder settings.
* *
* @return The configured Trie. * @return The configured Trie.
*/ */
public Trie build() { public Trie build() {
getTrie().constructFailureStates(); this.trie.constructFailureStates();
return getTrie();
}
private Trie getTrie() {
return this.trie; return this.trie;
} }
private TrieConfig getTrieConfig() {
return this.trieConfig;
}
/**
* @deprecated Use onlyFirstMatch()
*/
public TrieBuilder stopOnHit() {
return onlyFirstMatch();
}
/** /**
* @deprecated Use ignoreCase() * @deprecated Use ignoreCase()
*
* @return This builder.
*/ */
public TrieBuilder caseInsensitive() { public TrieBuilder caseInsensitive() {
return ignoreCase(); return ignoreCase();
@ -424,6 +430,8 @@ public class Trie {
/** /**
* @deprecated Use ignoreOverlaps() * @deprecated Use ignoreOverlaps()
*
* @return This builder.
*/ */
public TrieBuilder removeOverlaps() { public TrieBuilder removeOverlaps() {
return ignoreOverlaps(); return ignoreOverlaps();