Marmelator/aho-corasick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated source base to leverage JDK 1.7 syntax. Added more final modifiers. Eliminated parameter modification inside method. Some formatting. Changed TrieBuilder to offer CharSequence instead of String; revised Trie accordingly. Removed some duplication. NetBeans automatically translated the code to use static imports (as per JDK 1.7 syntax).

Browse Source
This commit is contained in:
djarvis 2016-11-29 18:38:00 -08:00 committed by robert-bor
parent a89a6bac95
commit 503a0f1c76
15 changed files with 292 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/org/ahocorasick/interval/IntervalNode.java
View File

@ -8,16 +8,16 @@ public class IntervalNode {
private enum Direction {LEFT, RIGHT}
private IntervalNode left = null;
private IntervalNode right = null;
private IntervalNode left;
private IntervalNode right;
private int point;
private List<Intervalable> intervals = new ArrayList<Intervalable>();
private List<Intervalable> intervals = new ArrayList<>();
public IntervalNode(List<Intervalable> intervals) {
public IntervalNode(final List<Intervalable> intervals) {
this.point = determineMedian(intervals);
List<Intervalable> toLeft = new ArrayList<Intervalable>();
List<Intervalable> toRight = new ArrayList<Intervalable>();
final List<Intervalable> toLeft = new ArrayList<>();
final List<Intervalable> toRight = new ArrayList<>();
for (Intervalable interval : intervals) {
if (interval.getEnd() < this.point) {
@ -37,7 +37,7 @@ public class IntervalNode {
}
}
public int determineMedian(List<Intervalable> intervals) {
public int determineMedian(final List<Intervalable> intervals) {
int start = -1;
int end = -1;
for (Intervalable interval : intervals) {
@ -53,17 +53,19 @@ public class IntervalNode {
return (start + end) / 2;
}
public List<Intervalable> findOverlaps(Intervalable interval) {
public List<Intervalable> findOverlaps(final Intervalable interval) {
final List<Intervalable> overlaps = new ArrayList<>();
List<Intervalable> overlaps = new ArrayList<Intervalable>();
if (this.point < interval.getStart()) { // Tends to the right
if (this.point < interval.getStart()) {
// Tends to the right
addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval));
addToOverlaps(interval, overlaps, checkForOverlapsToTheRight(interval));
} else if (this.point > interval.getEnd()) { // Tends to the left
} else if (this.point > interval.getEnd()) {
// Tends to the left
addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval));
addToOverlaps(interval, overlaps, checkForOverlapsToTheLeft(interval));
} else { // Somewhere in the middle
} else {
// Somewhere in the middle
addToOverlaps(interval, overlaps, this.intervals);
addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval));
addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval));
@ -72,26 +74,30 @@ public class IntervalNode {
return overlaps;
}
protected void addToOverlaps(Intervalable interval, List<Intervalable> overlaps, List<Intervalable> newOverlaps) {
for (Intervalable currentInterval : newOverlaps) {
protected void addToOverlaps(
final Intervalable interval,
final List<Intervalable> overlaps,
final List<Intervalable> newOverlaps) {
for (final Intervalable currentInterval : newOverlaps) {
if (!currentInterval.equals(interval)) {
overlaps.add(currentInterval);
}
}
}
protected List<Intervalable> checkForOverlapsToTheLeft(Intervalable interval) {
protected List<Intervalable> checkForOverlapsToTheLeft(final Intervalable interval) {
return checkForOverlaps(interval, Direction.LEFT);
}
protected List<Intervalable> checkForOverlapsToTheRight(Intervalable interval) {
protected List<Intervalable> checkForOverlapsToTheRight(final Intervalable interval) {
return checkForOverlaps(interval, Direction.RIGHT);
}
protected List<Intervalable> checkForOverlaps(Intervalable interval, Direction direction) {
protected List<Intervalable> checkForOverlaps(
final Intervalable interval, final Direction direction) {
final List<Intervalable> overlaps = new ArrayList<>();
List<Intervalable> overlaps = new ArrayList<Intervalable>();
for (Intervalable currentInterval : this.intervals) {
for (final Intervalable currentInterval : this.intervals) {
switch (direction) {
case LEFT:
if (currentInterval.getStart() <= interval.getEnd()) {
@ -105,15 +111,13 @@ public class IntervalNode {
break;
}
}
return overlaps;
}
protected List<Intervalable> findOverlappingRanges(IntervalNode node, Intervalable interval) {
if (node != null) {
return node.findOverlaps(interval);
}
return Collections.emptyList();
return node == null
? Collections.<Intervalable>emptyList()
: node.findOverlaps( interval );
}
}

18
src/main/java/org/ahocorasick/interval/IntervalTree.java
View File

@ -1,26 +1,26 @@
package org.ahocorasick.interval;
import java.util.Collections;
import static java.util.Collections.sort;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
public class IntervalTree {
private IntervalNode rootNode = null;
private final IntervalNode rootNode;
public IntervalTree(List<Intervalable> intervals) {
this.rootNode = new IntervalNode(intervals);
}
public List<Intervalable> removeOverlaps(List<Intervalable> intervals) {
public List<Intervalable> removeOverlaps(final List<Intervalable> intervals) {
// Sort the intervals on size, then left-most position
Collections.sort(intervals, new IntervalableComparatorBySize());
sort(intervals, new IntervalableComparatorBySize());
Set<Intervalable> removeIntervals = new TreeSet<Intervalable>();
final Set<Intervalable> removeIntervals = new TreeSet<>();
for (Intervalable interval : intervals) {
for (final Intervalable interval : intervals) {
// If the interval was already removed, ignore it
if (removeIntervals.contains(interval)) {
continue;
@ -31,17 +31,17 @@ public class IntervalTree {
}
// Remove all intervals that were overlapping
for (Intervalable removeInterval : removeIntervals) {
for (final Intervalable removeInterval : removeIntervals) {
intervals.remove(removeInterval);
}
// Sort the intervals, now on left-most position only
Collections.sort(intervals, new IntervalableComparatorByPosition());
sort(intervals, new IntervalableComparatorByPosition());
return intervals;
}
public List<Intervalable> findOverlaps(Intervalable interval) {
public List<Intervalable> findOverlaps(final Intervalable interval) {
return rootNode.findOverlaps(interval);
}

1
src/main/java/org/ahocorasick/interval/Intervalable.java
View File

@ -7,5 +7,4 @@ public interface Intervalable extends Comparable {
public int getEnd();
public int size();
}

2
src/main/java/org/ahocorasick/interval/IntervalableComparatorByPosition.java
View File

@ -5,7 +5,7 @@ import java.util.Comparator;
public class IntervalableComparatorByPosition implements Comparator<Intervalable> {
@Override
public int compare(Intervalable intervalable, Intervalable intervalable2) {
public int compare(final Intervalable intervalable, final Intervalable intervalable2) {
return intervalable.getStart() - intervalable2.getStart();
}

4
src/main/java/org/ahocorasick/interval/IntervalableComparatorBySize.java
View File

@ -5,11 +5,13 @@ import java.util.Comparator;
public class IntervalableComparatorBySize implements Comparator<Intervalable> {
@Override
public int compare(Intervalable intervalable, Intervalable intervalable2) {
public int compare(final Intervalable intervalable, final Intervalable intervalable2) {
int comparison = intervalable2.size() - intervalable.size();
if (comparison == 0) {
comparison = intervalable.getStart() - intervalable2.getStart();
}
return comparison;
}

1
src/main/java/org/ahocorasick/trie/Emit.java
View File

@ -20,5 +20,4 @@ public class Emit extends Interval implements Intervalable {
public String toString() {
return super.toString() + "=" + this.keyword;
}
}

5
src/main/java/org/ahocorasick/trie/MatchToken.java
View File

@ -2,9 +2,9 @@ package org.ahocorasick.trie;
public class MatchToken extends Token {
private Emit emit;
private final Emit emit;
public MatchToken(String fragment, Emit emit) {
public MatchToken(final String fragment, final Emit emit) {
super(fragment);
this.emit = emit;
}
@ -18,5 +18,4 @@ public class MatchToken extends Token {
public Emit getEmit() {
return this.emit;
}
}

64
src/main/java/org/ahocorasick/trie/State.java
View File

@ -4,51 +4,43 @@ import java.util.*;
/**
* <p>
* A state has various important tasks it must attend to:
* A state has various important tasks it must attend to:
* </p>
* <p>
*
* <ul>
* <li>success; when a character points to another state, it must return that state</li>
* <li>failure; when a character has no matching state, the algorithm must be able to fall back on a
* state with less depth</li>
* <li>emits; when this state is passed and keywords have been matched, the matches must be
* 'emitted' so that they can be used later on.</li>
* <li>success; when a character points to another state, it must return that state</li>
* <li>failure; when a character has no matching state, the algorithm must be able to fall back on a
* state with less depth</li>
* <li>emits; when this state is passed and keywords have been matched, the matches must be
* 'emitted' so that they can be used later on.</li>
* </ul>
*
* <p>
* <p>
* The root state is special in the sense that it has no failure state; it cannot fail. If it 'fails'
* it will still parse the next character and start from the root node. This ensures that the algorithm
* always runs. All other states always have a fail state.
* The root state is special in the sense that it has no failure state; it cannot fail. If it 'fails'
* it will still parse the next character and start from the root node. This ensures that the algorithm
* always runs. All other states always have a fail state.
* </p>
*
* @author Robert Bor
*/
public class State {
/**
* effective the size of the keyword
*/
/** effective the size of the keyword */
private final int depth;
/**
* only used for the root state to refer to itself in case no matches have been found
*/
/** only used for the root state to refer to itself in case no matches have been found */
private final State rootState;
/**
* referred to in the white paper as the 'goto' structure. From a state it is possible to go
* to other states, depending on the character passed.
*/
private final Map<Character, State> success = new HashMap<>();
private final Map<Character,State> success = new HashMap<>();
/**
* if no matching states are found, the failure state will be returned
*/
/** if no matching states are found, the failure state will be returned */
private State failure;
/**
* whenever this state is reached, it will emit the matches keywords for future reference
*/
/** whenever this state is reached, it will emit the matches keywords for future reference */
private Set<String> emits;
public State() {
@ -74,24 +66,24 @@ public class State {
return nextState(character, false);
}
public State nextStateIgnoreRootState(Character character) {
public State nextStateIgnoreRootState(final Character character) {
return nextState(character, true);
}
public State addState(String keyword) {
State state = this;
public State addState(final String keyword ) {
State state = this;
for (final Character character : keyword.toCharArray()) {
state = state.addState(character);
}
for (final Character character : keyword.toCharArray()) {
state = state.addState(character);
}
return state;
return state;
}
public State addState(Character character) {
public State addState(final Character character) {
State nextState = nextStateIgnoreRootState(character);
if (nextState == null) {
nextState = new State(this.depth + 1);
nextState = new State(this.depth+1);
this.success.put(character, nextState);
}
return nextState;
@ -101,21 +93,21 @@ public class State {
return this.depth;
}
public void addEmit(String keyword) {
public void addEmit(final String keyword) {
if (this.emits == null) {
this.emits = new TreeSet<>();
}
this.emits.add(keyword);
}
public void addEmit(Collection<String> emits) {
public void addEmit(final Collection<String> emits) {
for (String emit : emits) {
addEmit(emit);
}
}
public Collection<String> emit() {
return this.emits == null ? Collections.<String>emptyList() : this.emits;
return this.emits == null ? Collections.<String> emptyList() : this.emits;
}
public State failure() {

161
src/main/java/org/ahocorasick/trie/Trie.java
View File

@ -1,13 +1,13 @@
package org.ahocorasick.trie;
import static java.lang.Character.isAlphabetic;
import static java.lang.Character.isWhitespace;
import static java.lang.Character.toLowerCase;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingDeque;
import org.ahocorasick.interval.IntervalTree;
import org.ahocorasick.interval.Intervalable;
import org.ahocorasick.trie.handler.DefaultEmitHandler;
@ -34,44 +34,21 @@ public class Trie {
* Used by the builder to add a text search keyword.
*
* @param keyword The search term to add to the list of search terms.
*
* @throws NullPointerException if the keyword is null.
*/
private void addKeyword(String keyword) {
if (keyword.isEmpty()) {
return;
}
private void addKeyword( String keyword ) {
if( keyword.length() > 0 ) {
if( isCaseInsensitive() ) {
keyword = keyword.toLowerCase();
}
if (isCaseInsensitive()) {
keyword = keyword.toLowerCase();
}
addState(keyword).addEmit(keyword);
}
/**
* Delegates to addKeyword.
*
* @param keywords List of search term to add to the list of search terms.
*/
private void addKeywords(final String[] keywords) {
for (final String keyword : keywords) {
addKeyword(keyword);
addState( keyword ).addEmit( keyword );
}
}
/**
* Delegates to addKeyword.
*
* @param keywords List of search term to add to the list of search terms.
*/
private void addKeywords(final Collection<String> keywords) {
for (final String keyword : keywords) {
addKeyword(keyword);
}
}
private State addState(final String keyword) {
return getRootState().addState(keyword);
private State addState( final String keyword ) {
return getRootState().addState( keyword );
}
public Collection<Token> tokenize(final String text) {
@ -95,12 +72,15 @@ public class Trie {
return tokens;
}
private Token createFragment(final Emit emit, final String text, final int lastCollectedPosition) {
return new FragmentToken(text.substring(lastCollectedPosition + 1, emit == null ? text.length() : emit.getStart()));
private Token createFragment(
final Emit emit,
final String text,
final int lastCollectedPosition) {
return new FragmentToken(text.substring(lastCollectedPosition+1, emit == null ? text.length() : emit.getStart()));
}
private Token createMatch(Emit emit, String text) {
return new MatchToken(text.substring(emit.getStart(), emit.getEnd() + 1), emit);
private Token createMatch(final Emit emit, final String text) {
return new MatchToken(text.substring(emit.getStart(), emit.getEnd()+1), emit);
}
@SuppressWarnings("unchecked")
@ -119,7 +99,7 @@ public class Trie {
}
if (!trieConfig.isAllowOverlaps()) {
IntervalTree intervalTree = new IntervalTree((List<Intervalable>) (List<?>) collectedEmits);
IntervalTree intervalTree = new IntervalTree((List<Intervalable>)(List<?>)collectedEmits);
intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
}
@ -138,7 +118,7 @@ public class Trie {
// TODO: Maybe lowercase the entire string at once?
if (trieConfig.isCaseInsensitive()) {
character = Character.toLowerCase(character);
character = toLowerCase(character);
}
currentState = getState(currentState, character);
@ -164,14 +144,14 @@ public class Trie {
// TODO: Lowercase the entire string at once?
if (trieConfig.isCaseInsensitive()) {
character = Character.toLowerCase(character);
character = toLowerCase(character);
}
currentState = getState(currentState, character);
Collection<String> emitStrs = currentState.emit();
if (emitStrs != null && !emitStrs.isEmpty()) {
for (String emitStr : emitStrs) {
for (final String emitStr : emitStrs) {
final Emit emit = new Emit(position - emitStr.length() + 1, position, emitStr);
if (trieConfig.isOnlyWholeWords()) {
if (!isPartialMatch(text, emit)) {
@ -190,9 +170,9 @@ public class Trie {
private boolean isPartialMatch(final CharSequence searchText, final Emit emit) {
return (emit.getStart() != 0 &&
Character.isAlphabetic(searchText.charAt(emit.getStart() - 1))) ||
(emit.getEnd() + 1 != searchText.length() &&
Character.isAlphabetic(searchText.charAt(emit.getEnd() + 1)));
isAlphabetic(searchText.charAt(emit.getStart() - 1))) ||
(emit.getEnd() + 1 != searchText.length() &&
isAlphabetic(searchText.charAt(emit.getEnd() + 1)));
}
private void removePartialMatches(final CharSequence searchText, final List<Emit> collectedEmits) {
@ -215,7 +195,7 @@ public class Trie {
for (final Emit emit : collectedEmits) {
if ((emit.getStart() == 0 || isWhitespace(searchText.charAt(emit.getStart() - 1))) &&
(emit.getEnd() + 1 == size || isWhitespace(searchText.charAt(emit.getEnd() + 1)))) {
(emit.getEnd() + 1 == size || isWhitespace(searchText.charAt(emit.getEnd() + 1)))) {
continue;
}
removeEmits.add(emit);
@ -226,15 +206,16 @@ public class Trie {
}
}
private State getState(State currentState, final Character character) {
State newCurrentState = currentState.nextState(character);
private State getState(final State initialState, final Character character) {
State currentState = initialState;
State updatedState = currentState.nextState(character);
while (newCurrentState == null) {
while (updatedState == null) {
currentState = currentState.failure();
newCurrentState = currentState.nextState(character);
updatedState = currentState.nextState(character);
}
return newCurrentState;
return updatedState;
}
private void constructFailureStates() {
@ -267,7 +248,10 @@ public class Trie {
}
}
private boolean storeEmits(final int position, final State currentState, final EmitHandler emitHandler) {
private boolean storeEmits(
final int position,
final State currentState,
final EmitHandler emitHandler) {
boolean emitted = false;
final Collection<String> emits = currentState.emit();
@ -283,15 +267,16 @@ public class Trie {
}
private boolean isCaseInsensitive() {
return trieConfig.isCaseInsensitive();
return trieConfig.isCaseInsensitive();
}
private State getRootState() {
return this.rootState;
return this.rootState;
}
/**
* Provides a fluent interface for constructing Trie instances.
* Constructs a TrieBuilder instance for configuring the Trie using a fluent
* interface.
*
* @return The builder used to configure its Trie.
*/
@ -299,6 +284,9 @@ public class Trie {
return new TrieBuilder();
}
/**
* Provides a fluent interface for constructing Trie instances.
*/
public static class TrieBuilder {
private final TrieConfig trieConfig = new TrieConfig();
@ -308,18 +296,18 @@ public class Trie {
/**
* Default (empty) constructor.
*/
private TrieBuilder() {
}
private TrieBuilder() {}
/**
* Adds a keyword to the Trie's list of text search keywords.
*
* @param keyword The keyword to add to the list.
*
* @return This builder.
* @throws NullPointerException if the keyword is null.
*/
public TrieBuilder addKeyword(final String keyword) {
this.trie.addKeyword(keyword);
public TrieBuilder addKeyword(final CharSequence keyword) {
getTrie().addKeyword( keyword.toString() );
return this;
}
@ -327,10 +315,14 @@ public class Trie {
* Adds a list of keywords to the Trie's list of text search keywords.
*
* @param keywords The keywords to add to the list.
*
* @return This builder.
*/
public TrieBuilder addKeywords(final String... keywords) {
this.trie.addKeywords(keywords);
public TrieBuilder addKeywords(final CharSequence... keywords) {
for( final CharSequence keyword : keywords ) {
addKeyword( keyword );
}
return this;
}
@ -338,21 +330,21 @@ public class Trie {
* Adds a list of keywords to the Trie's list of text search keywords.
*
* @param keywords The keywords to add to the list.
*
* @return This builder.
*/
public TrieBuilder addKeywords(final Collection<String> keywords) {
this.trie.addKeywords(keywords);
return this;
public TrieBuilder addKeywords(final Collection<CharSequence> keywords) {
return addKeywords( keywords.toArray( new CharSequence[ keywords.size() ] ) );
}
/**
* Configure the Trie to ignore case when searching for keywords in
* the text.
* Configure the Trie to ignore case when searching for keywords in the
* text.
*
* @return This builder.
*/
public TrieBuilder ignoreCase() {
this.trieConfig.setCaseInsensitive(true);
getTrieConfig().setCaseInsensitive(true);
return this;
}
@ -362,7 +354,7 @@ public class Trie {
* @return This builder.
*/
public TrieBuilder ignoreOverlaps() {
this.trieConfig.setAllowOverlaps(false);
getTrieConfig().setAllowOverlaps(false);
return this;
}
@ -372,7 +364,7 @@ public class Trie {
* @return This builder.
*/
public TrieBuilder onlyWholeWords() {
this.trieConfig.setOnlyWholeWords(true);
getTrieConfig().setOnlyWholeWords(true);
return this;
}
@ -384,33 +376,47 @@ public class Trie {
* @return This builder.
*/
public TrieBuilder onlyWholeWordsWhiteSpaceSeparated() {
this.trieConfig.setOnlyWholeWordsWhiteSpaceSeparated(true);
getTrieConfig().setOnlyWholeWordsWhiteSpaceSeparated(true);
return this;
}
/**
* Configure the Trie to stop after the first keyword is found in the
* text.
* Configure the Trie to stop searching for matches after the first
* keyword is found in the text.
*
* @return This builder.
*/
public TrieBuilder stopOnHit() {
trie.trieConfig.setStopOnHit(true);
public TrieBuilder onlyFirstMatch() {
getTrieConfig().setStopOnHit(true);
return this;
}
/**
* Configure the Trie based on the builder settings.
* Construct the Trie using the builder settings.
*
* @return The configured Trie.
*/
public Trie build() {
this.trie.constructFailureStates();
getTrie().constructFailureStates();
return getTrie();
}
private Trie getTrie() {
return this.trie;
}
private TrieConfig getTrieConfig() {
return this.trieConfig;
}
/**
* @deprecated Use onlyFirstMatch()
*/
public TrieBuilder stopOnHit() {
return onlyFirstMatch();
}
/**
* @return This builder.
* @deprecated Use ignoreCase()
*/
public TrieBuilder caseInsensitive() {
@ -418,7 +424,6 @@ public class Trie {
}
/**
* @return This builder.
* @deprecated Use ignoreOverlaps()
*/
public TrieBuilder removeOverlaps() {

8
src/main/java/org/ahocorasick/trie/handler/DefaultEmitHandler.java
View File

@ -1,21 +1,19 @@
package org.ahocorasick.trie.handler;
import org.ahocorasick.trie.Emit;
import java.util.ArrayList;
import java.util.List;
import org.ahocorasick.trie.Emit;
public class DefaultEmitHandler implements EmitHandler {
private List<Emit> emits = new ArrayList<>();
private final List<Emit> emits = new ArrayList<>();
@Override
public void emit(Emit emit) {
public void emit(final Emit emit) {
this.emits.add(emit);
}
public List<Emit> getEmits() {
return this.emits;
}
}

2
src/test/java/org/ahocorasick/interval/IntervalTest.java
View File

@ -44,7 +44,7 @@ public class IntervalTest {
@Test
public void comparable() {
Set<Interval> intervals = new TreeSet<Interval>();
Set<Interval> intervals = new TreeSet<>();
intervals.add(new Interval(4, 6));
intervals.add(new Interval(2, 7));
intervals.add(new Interval(3, 4));

4
src/test/java/org/ahocorasick/interval/IntervalTreeTest.java
View File

@ -12,7 +12,7 @@ public class IntervalTreeTest {
@Test
public void findOverlaps() {
List<Intervalable> intervals = new ArrayList<Intervalable>();
List<Intervalable> intervals = new ArrayList<>();
intervals.add(new Interval(0, 2));
intervals.add(new Interval(1, 3));
intervals.add(new Interval(2, 4));
@ -30,7 +30,7 @@ public class IntervalTreeTest {
@Test
public void removeOverlaps() {
List<Intervalable> intervals = new ArrayList<Intervalable>();
List<Intervalable> intervals = new ArrayList<>();
intervals.add(new Interval(0, 2));
intervals.add(new Interval(4, 5));
intervals.add(new Interval(2, 10));

11
src/test/java/org/ahocorasick/interval/IntervalableComparatorByPositionTest.java
View File

@ -4,6 +4,7 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
import static java.util.Collections.sort;
import java.util.List;
import static junit.framework.Assert.assertEquals;
@ -12,11 +13,11 @@ public class IntervalableComparatorByPositionTest {
@Test
public void sortOnPosition() {
List<Intervalable> intervals = new ArrayList<Intervalable>();
intervals.add(new Interval(4, 5));
intervals.add(new Interval(1, 4));
intervals.add(new Interval(3, 8));
Collections.sort(intervals, new IntervalableComparatorByPosition());
List<Intervalable> intervals = new ArrayList<>();
intervals.add(new Interval(4,5));
intervals.add(new Interval(1,4));
intervals.add(new Interval(3,8));
sort(intervals, new IntervalableComparatorByPosition());
assertEquals(4, intervals.get(0).size());
assertEquals(6, intervals.get(1).size());
assertEquals(2, intervals.get(2).size());

20
src/test/java/org/ahocorasick/interval/IntervalableComparatorBySizeTest.java
View File

@ -4,6 +4,8 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
import static java.util.Collections.sort;
import static java.util.Collections.sort;
import java.util.List;
import static junit.framework.Assert.assertEquals;
@ -12,11 +14,11 @@ public class IntervalableComparatorBySizeTest {
@Test
public void sortOnSize() {
List<Intervalable> intervals = new ArrayList<Intervalable>();
intervals.add(new Interval(4, 5));
intervals.add(new Interval(1, 4));
intervals.add(new Interval(3, 8));
Collections.sort(intervals, new IntervalableComparatorBySize());
List<Intervalable> intervals = new ArrayList<>();
intervals.add(new Interval(4,5));
intervals.add(new Interval(1,4));
intervals.add(new Interval(3,8));
sort(intervals, new IntervalableComparatorBySize());
assertEquals(6, intervals.get(0).size());
assertEquals(4, intervals.get(1).size());
assertEquals(2, intervals.get(2).size());
@ -24,10 +26,10 @@ public class IntervalableComparatorBySizeTest {
@Test
public void sortOnSizeThenPosition() {
List<Intervalable> intervals = new ArrayList<Intervalable>();
intervals.add(new Interval(4, 7));
intervals.add(new Interval(2, 5));
Collections.sort(intervals, new IntervalableComparatorBySize());
List<Intervalable> intervals = new ArrayList<>();
intervals.add(new Interval(4,7));
intervals.add(new Interval(2,5));
sort(intervals, new IntervalableComparatorBySize());
assertEquals(2, intervals.get(0).getStart());
assertEquals(4, intervals.get(1).getStart());
}

126
src/test/java/org/ahocorasick/trie/TrieTest.java
View File

@ -5,39 +5,37 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import static java.util.concurrent.ThreadLocalRandom.current;
import static junit.framework.Assert.assertEquals;
import static org.ahocorasick.trie.Trie.builder;
import org.ahocorasick.trie.handler.EmitHandler;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
public class TrieTest {
private final static String[] ALPHABET = new String[]{
"abc", "bcd", "cde"
"abc", "bcd", "cde"
};
private final static String[] PRONOUNS = new String[]{
"hers", "his", "she", "he"
"hers", "his", "she", "he"
};
private final static String[] FOOD = new String[]{
"veal", "cauliflower", "broccoli", "tomatoes"
"veal", "cauliflower", "broccoli", "tomatoes"
};
private final static String[] GREEK_LETTERS = new String[]{
"Alpha", "Beta", "Gamma"
"Alpha", "Beta", "Gamma"
};
private final static String[] UNICODE = new String[]{
"turning", "once", "again", "börkü"
"turning", "once", "again", "börkü"
};
@Test
public void keywordAndTextAreTheSame() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeyword(ALPHABET[0])
.build();
Collection<Emit> emits = trie.parseText(ALPHABET[0]);
@ -47,7 +45,7 @@ public class TrieTest {
@Test
public void keywordAndTextAreTheSameFirstMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeyword(ALPHABET[0])
.build();
Emit firstMatch = trie.firstMatch(ALPHABET[0]);
@ -56,7 +54,7 @@ public class TrieTest {
@Test
public void textIsLongerThanKeyword() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeyword(ALPHABET[0])
.build();
Collection<Emit> emits = trie.parseText(" " + ALPHABET[0]);
@ -66,7 +64,7 @@ public class TrieTest {
@Test
public void textIsLongerThanKeywordFirstMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeyword(ALPHABET[0])
.build();
Emit firstMatch = trie.firstMatch(" " + ALPHABET[0]);
@ -75,7 +73,7 @@ public class TrieTest {
@Test
public void variousKeywordsOneMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(ALPHABET)
.build();
Collection<Emit> emits = trie.parseText("bcd");
@ -85,7 +83,7 @@ public class TrieTest {
@Test
public void variousKeywordsFirstMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(ALPHABET)
.build();
Emit firstMatch = trie.firstMatch("bcd");
@ -94,7 +92,7 @@ public class TrieTest {
@Test
public void ushersTestAndStopOnHit() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(PRONOUNS)
.stopOnHit()
.build();
@ -107,7 +105,7 @@ public class TrieTest {
@Test
public void ushersTest() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(PRONOUNS)
.build();
Collection<Emit> emits = trie.parseText("ushers");
@ -120,7 +118,7 @@ public class TrieTest {
@Test
public void ushersTestWithCapitalKeywords() {
Trie trie = Trie.builder()
Trie trie = builder()
.ignoreCase()
.addKeyword("HERS")
.addKeyword("HIS")
@ -137,7 +135,7 @@ public class TrieTest {
@Test
public void ushersTestFirstMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(PRONOUNS)
.build();
Emit firstMatch = trie.firstMatch("ushers");
@ -146,7 +144,7 @@ public class TrieTest {
@Test
public void ushersTestByCallback() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(PRONOUNS)
.build();
@ -168,7 +166,7 @@ public class TrieTest {
@Test
public void misleadingTest() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeyword("hers")
.build();
Collection<Emit> emits = trie.parseText("h he her hers");
@ -178,7 +176,7 @@ public class TrieTest {
@Test
public void misleadingTestFirstMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeyword("hers")
.build();
Emit firstMatch = trie.firstMatch("h he her hers");
@ -187,7 +185,7 @@ public class TrieTest {
@Test
public void recipes() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(FOOD)
.build();
Collection<Emit> emits = trie.parseText("2 cauliflowers, 3 tomatoes, 4 slices of veal, 100g broccoli");
@ -200,7 +198,7 @@ public class TrieTest {
@Test
public void recipesFirstMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(FOOD)
.build();
Emit firstMatch = trie.firstMatch("2 cauliflowers, 3 tomatoes, 4 slices of veal, 100g broccoli");
@ -210,7 +208,7 @@ public class TrieTest {
@Test
public void longAndShortOverlappingMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeyword("he")
.addKeyword("hehehehe")
.build();
@ -227,7 +225,7 @@ public class TrieTest {
@Test
public void nonOverlapping() {
Trie trie = Trie.builder().removeOverlaps()
Trie trie = builder().removeOverlaps()
.addKeyword("ab")
.addKeyword("cba")
.addKeyword("ababc")
@ -242,7 +240,7 @@ public class TrieTest {
@Test
public void nonOverlappingFirstMatch() {
Trie trie = Trie.builder().removeOverlaps()
Trie trie = builder().removeOverlaps()
.addKeyword("ab")
.addKeyword("cba")
.addKeyword("ababc")
@ -254,7 +252,7 @@ public class TrieTest {
@Test
public void containsMatch() {
Trie trie = Trie.builder().removeOverlaps()
Trie trie = builder().removeOverlaps()
.addKeyword("ab")
.addKeyword("cba")
.addKeyword("ababc")
@ -264,7 +262,7 @@ public class TrieTest {
@Test
public void startOfChurchillSpeech() {
Trie trie = Trie.builder().removeOverlaps()
Trie trie = builder().removeOverlaps()
.addKeyword("T")
.addKeyword("u")
.addKeyword("ur")
@ -282,7 +280,7 @@ public class TrieTest {
@Test
public void partialMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.onlyWholeWords()
.addKeyword("sugar")
.build();
@ -293,7 +291,7 @@ public class TrieTest {
@Test
public void partialMatchFirstMatch() {
Trie trie = Trie.builder()
Trie trie = builder()
.onlyWholeWords()
.addKeyword("sugar")
.build();
@ -304,7 +302,7 @@ public class TrieTest {
@Test
public void tokenizeFullSentence() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(GREEK_LETTERS)
.build();
Collection<Token> tokens = trie.tokenize("Hear: Alpha team first, Beta from the rear, Gamma in reserve");
@ -322,7 +320,7 @@ public class TrieTest {
// @see https://github.com/robert-bor/aho-corasick/issues/5
@Test
public void testStringIndexOutOfBoundsException() {
Trie trie = Trie.builder().ignoreCase().onlyWholeWords()
Trie trie = builder().ignoreCase().onlyWholeWords()
.addKeywords(UNICODE)
.build();
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
@ -336,7 +334,7 @@ public class TrieTest {
@Test
public void testIgnoreCase() {
Trie trie = Trie.builder().ignoreCase()
Trie trie = builder().ignoreCase()
.addKeywords(UNICODE)
.build();
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
@ -350,7 +348,7 @@ public class TrieTest {
@Test
public void testIgnoreCaseFirstMatch() {
Trie trie = Trie.builder().ignoreCase()
Trie trie = builder().ignoreCase()
.addKeywords(UNICODE)
.build();
Emit firstMatch = trie.firstMatch("TurninG OnCe AgAiN BÖRKÜ");
@ -360,7 +358,7 @@ public class TrieTest {
@Test
public void tokenizeTokensInSequence() {
Trie trie = Trie.builder()
Trie trie = builder()
.addKeywords(GREEK_LETTERS)
.build();
Collection<Token> tokens = trie.tokenize("Alpha Beta Gamma");
@ -370,7 +368,7 @@ public class TrieTest {
// @see https://github.com/robert-bor/aho-corasick/issues/7
@Test
public void testZeroLength() {
Trie trie = Trie.builder().ignoreOverlaps().onlyWholeWords().ignoreCase()
Trie trie = builder().ignoreOverlaps().onlyWholeWords().ignoreCase()
.addKeyword("")
.build();
trie.tokenize("Try a natural lip and subtle bronzer to keep all the focus on those big bright eyes with NARS Eyeshadow Duo in Rated R And the winner is... Boots No7 Advanced Renewal Anti-ageing Glycolic Peel Kit ($25 amazon.com) won most-appealing peel.");
@ -381,7 +379,7 @@ public class TrieTest {
public void testUnicode1() {
String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char
assertEquals("THIS", target.substring(5, 9)); // Java does it the right way
Trie trie = Trie.builder().ignoreCase().onlyWholeWords()
Trie trie = builder().ignoreCase().onlyWholeWords()
.addKeyword("this")
.build();
Collection<Emit> emits = trie.parseText(target);
@ -394,7 +392,7 @@ public class TrieTest {
@Test
public void testUnicode2() {
String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char
Trie trie = Trie.builder()
Trie trie = builder()
.ignoreCase()
.onlyWholeWords()
.addKeyword("this")
@ -406,11 +404,11 @@ public class TrieTest {
@Test
public void testPartialMatchWhiteSpaces() {
Trie trie = Trie.builder()
Trie trie = builder()
.onlyWholeWordsWhiteSpaceSeparated()
.addKeyword("#sugar-123")
.build();
Collection<Emit> emits = trie.parseText("#sugar-123 #sugar-1234"); // left, middle, right test
Collection < Emit > emits = trie.parseText("#sugar-123 #sugar-1234"); // left, middle, right test
assertEquals(1, emits.size()); // Match must not be made
checkEmit(emits.iterator().next(), 0, 9, "#sugar-123");
}
@ -419,19 +417,19 @@ public class TrieTest {
public void testLargeString() {
final int interval = 100;
final int textSize = 1000000;
final String keyword = FOOD[1];
final StringBuilder text = randomNumbers(textSize);
final String keyword = FOOD[ 1 ];
final StringBuilder text = randomNumbers( textSize );
injectKeyword(text, keyword, interval);
injectKeyword( text, keyword, interval );
Trie trie = Trie.builder()
.onlyWholeWords()
.addKeyword(keyword)
.build();
Trie trie = builder()
.onlyWholeWords()
.addKeyword( keyword )
.build();
final Collection<Emit> emits = trie.parseText(text);
final Collection<Emit> emits = trie.parseText( text );
assertEquals(textSize / interval, emits.size());
assertEquals( textSize / interval, emits.size() );
}
/**
@ -440,11 +438,11 @@ public class TrieTest {
* @param count The number of numbers to generate.
* @return A character sequence filled with random digits.
*/
private StringBuilder randomNumbers(int count) {
final StringBuilder sb = new StringBuilder(count);
private StringBuilder randomNumbers( final int count ) {
final StringBuilder sb = new StringBuilder( count );
while (--count > 0) {
sb.append(randomInt(0, 10));
for( int i = count - 1; i >= 0; i-- ) {
sb.append( randomInt( 0, 10 ) );
}
return sb;
@ -453,23 +451,23 @@ public class TrieTest {
/**
* Injects keywords into a string builder.
*
* @param source Should contain a bunch of random data that cannot match
* any keyword.
* @param keyword A keyword to inject repeatedly in the text.
* @param source Should contain a bunch of random data that cannot match
* any keyword.
* @param keyword A keyword to inject repeatedly in the text.
* @param interval How often to inject the keyword.
*/
private void injectKeyword(
final StringBuilder source,
final String keyword,
final int interval) {
final StringBuilder source,
final String keyword,
final int interval ) {
final int length = source.length();
for (int i = 0; i < length; i += interval) {
source.replace(i, i + keyword.length(), keyword);
for( int i = 0; i < length; i += interval ) {
source.replace( i, i + keyword.length(), keyword );
}
}
private int randomInt(final int min, final int max) {
return ThreadLocalRandom.current().nextInt(min, max);
private int randomInt( final int min, final int max ) {
return current().nextInt( min, max );
}
private void checkEmit(Emit next, int expectedStart, int expectedEnd, String expectedKeyword) {