Updated source base to leverage JDK 1.7 syntax. Added more final modifiers. Eliminated parameter modification inside method. Some formatting. Changed TrieBuilder to offer CharSequence instead of String; revised Trie accordingly. Removed some duplication. NetBeans automatically translated the code to use static imports (as per JDK 1.7 syntax).
This commit is contained in:
parent
a89a6bac95
commit
503a0f1c76
@ -8,16 +8,16 @@ public class IntervalNode {
|
|||||||
|
|
||||||
private enum Direction {LEFT, RIGHT}
|
private enum Direction {LEFT, RIGHT}
|
||||||
|
|
||||||
private IntervalNode left = null;
|
private IntervalNode left;
|
||||||
private IntervalNode right = null;
|
private IntervalNode right;
|
||||||
private int point;
|
private int point;
|
||||||
private List<Intervalable> intervals = new ArrayList<Intervalable>();
|
private List<Intervalable> intervals = new ArrayList<>();
|
||||||
|
|
||||||
public IntervalNode(List<Intervalable> intervals) {
|
public IntervalNode(final List<Intervalable> intervals) {
|
||||||
this.point = determineMedian(intervals);
|
this.point = determineMedian(intervals);
|
||||||
|
|
||||||
List<Intervalable> toLeft = new ArrayList<Intervalable>();
|
final List<Intervalable> toLeft = new ArrayList<>();
|
||||||
List<Intervalable> toRight = new ArrayList<Intervalable>();
|
final List<Intervalable> toRight = new ArrayList<>();
|
||||||
|
|
||||||
for (Intervalable interval : intervals) {
|
for (Intervalable interval : intervals) {
|
||||||
if (interval.getEnd() < this.point) {
|
if (interval.getEnd() < this.point) {
|
||||||
@ -37,7 +37,7 @@ public class IntervalNode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public int determineMedian(List<Intervalable> intervals) {
|
public int determineMedian(final List<Intervalable> intervals) {
|
||||||
int start = -1;
|
int start = -1;
|
||||||
int end = -1;
|
int end = -1;
|
||||||
for (Intervalable interval : intervals) {
|
for (Intervalable interval : intervals) {
|
||||||
@ -53,17 +53,19 @@ public class IntervalNode {
|
|||||||
return (start + end) / 2;
|
return (start + end) / 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Intervalable> findOverlaps(Intervalable interval) {
|
public List<Intervalable> findOverlaps(final Intervalable interval) {
|
||||||
|
final List<Intervalable> overlaps = new ArrayList<>();
|
||||||
|
|
||||||
List<Intervalable> overlaps = new ArrayList<Intervalable>();
|
if (this.point < interval.getStart()) {
|
||||||
|
// Tends to the right
|
||||||
if (this.point < interval.getStart()) { // Tends to the right
|
|
||||||
addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval));
|
addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval));
|
||||||
addToOverlaps(interval, overlaps, checkForOverlapsToTheRight(interval));
|
addToOverlaps(interval, overlaps, checkForOverlapsToTheRight(interval));
|
||||||
} else if (this.point > interval.getEnd()) { // Tends to the left
|
} else if (this.point > interval.getEnd()) {
|
||||||
|
// Tends to the left
|
||||||
addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval));
|
addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval));
|
||||||
addToOverlaps(interval, overlaps, checkForOverlapsToTheLeft(interval));
|
addToOverlaps(interval, overlaps, checkForOverlapsToTheLeft(interval));
|
||||||
} else { // Somewhere in the middle
|
} else {
|
||||||
|
// Somewhere in the middle
|
||||||
addToOverlaps(interval, overlaps, this.intervals);
|
addToOverlaps(interval, overlaps, this.intervals);
|
||||||
addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval));
|
addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval));
|
||||||
addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval));
|
addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval));
|
||||||
@ -72,26 +74,30 @@ public class IntervalNode {
|
|||||||
return overlaps;
|
return overlaps;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void addToOverlaps(Intervalable interval, List<Intervalable> overlaps, List<Intervalable> newOverlaps) {
|
protected void addToOverlaps(
|
||||||
for (Intervalable currentInterval : newOverlaps) {
|
final Intervalable interval,
|
||||||
|
final List<Intervalable> overlaps,
|
||||||
|
final List<Intervalable> newOverlaps) {
|
||||||
|
for (final Intervalable currentInterval : newOverlaps) {
|
||||||
if (!currentInterval.equals(interval)) {
|
if (!currentInterval.equals(interval)) {
|
||||||
overlaps.add(currentInterval);
|
overlaps.add(currentInterval);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<Intervalable> checkForOverlapsToTheLeft(Intervalable interval) {
|
protected List<Intervalable> checkForOverlapsToTheLeft(final Intervalable interval) {
|
||||||
return checkForOverlaps(interval, Direction.LEFT);
|
return checkForOverlaps(interval, Direction.LEFT);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<Intervalable> checkForOverlapsToTheRight(Intervalable interval) {
|
protected List<Intervalable> checkForOverlapsToTheRight(final Intervalable interval) {
|
||||||
return checkForOverlaps(interval, Direction.RIGHT);
|
return checkForOverlaps(interval, Direction.RIGHT);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<Intervalable> checkForOverlaps(Intervalable interval, Direction direction) {
|
protected List<Intervalable> checkForOverlaps(
|
||||||
|
final Intervalable interval, final Direction direction) {
|
||||||
|
final List<Intervalable> overlaps = new ArrayList<>();
|
||||||
|
|
||||||
List<Intervalable> overlaps = new ArrayList<Intervalable>();
|
for (final Intervalable currentInterval : this.intervals) {
|
||||||
for (Intervalable currentInterval : this.intervals) {
|
|
||||||
switch (direction) {
|
switch (direction) {
|
||||||
case LEFT:
|
case LEFT:
|
||||||
if (currentInterval.getStart() <= interval.getEnd()) {
|
if (currentInterval.getStart() <= interval.getEnd()) {
|
||||||
@ -105,15 +111,13 @@ public class IntervalNode {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return overlaps;
|
return overlaps;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
protected List<Intervalable> findOverlappingRanges(IntervalNode node, Intervalable interval) {
|
protected List<Intervalable> findOverlappingRanges(IntervalNode node, Intervalable interval) {
|
||||||
if (node != null) {
|
return node == null
|
||||||
return node.findOverlaps(interval);
|
? Collections.<Intervalable>emptyList()
|
||||||
|
: node.findOverlaps( interval );
|
||||||
}
|
}
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,26 +1,26 @@
|
|||||||
package org.ahocorasick.interval;
|
package org.ahocorasick.interval;
|
||||||
|
|
||||||
import java.util.Collections;
|
import static java.util.Collections.sort;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
|
|
||||||
public class IntervalTree {
|
public class IntervalTree {
|
||||||
|
|
||||||
private IntervalNode rootNode = null;
|
private final IntervalNode rootNode;
|
||||||
|
|
||||||
public IntervalTree(List<Intervalable> intervals) {
|
public IntervalTree(List<Intervalable> intervals) {
|
||||||
this.rootNode = new IntervalNode(intervals);
|
this.rootNode = new IntervalNode(intervals);
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Intervalable> removeOverlaps(List<Intervalable> intervals) {
|
public List<Intervalable> removeOverlaps(final List<Intervalable> intervals) {
|
||||||
|
|
||||||
// Sort the intervals on size, then left-most position
|
// Sort the intervals on size, then left-most position
|
||||||
Collections.sort(intervals, new IntervalableComparatorBySize());
|
sort(intervals, new IntervalableComparatorBySize());
|
||||||
|
|
||||||
Set<Intervalable> removeIntervals = new TreeSet<Intervalable>();
|
final Set<Intervalable> removeIntervals = new TreeSet<>();
|
||||||
|
|
||||||
for (Intervalable interval : intervals) {
|
for (final Intervalable interval : intervals) {
|
||||||
// If the interval was already removed, ignore it
|
// If the interval was already removed, ignore it
|
||||||
if (removeIntervals.contains(interval)) {
|
if (removeIntervals.contains(interval)) {
|
||||||
continue;
|
continue;
|
||||||
@ -31,17 +31,17 @@ public class IntervalTree {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Remove all intervals that were overlapping
|
// Remove all intervals that were overlapping
|
||||||
for (Intervalable removeInterval : removeIntervals) {
|
for (final Intervalable removeInterval : removeIntervals) {
|
||||||
intervals.remove(removeInterval);
|
intervals.remove(removeInterval);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sort the intervals, now on left-most position only
|
// Sort the intervals, now on left-most position only
|
||||||
Collections.sort(intervals, new IntervalableComparatorByPosition());
|
sort(intervals, new IntervalableComparatorByPosition());
|
||||||
|
|
||||||
return intervals;
|
return intervals;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Intervalable> findOverlaps(Intervalable interval) {
|
public List<Intervalable> findOverlaps(final Intervalable interval) {
|
||||||
return rootNode.findOverlaps(interval);
|
return rootNode.findOverlaps(interval);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -7,5 +7,4 @@ public interface Intervalable extends Comparable {
|
|||||||
public int getEnd();
|
public int getEnd();
|
||||||
|
|
||||||
public int size();
|
public int size();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,7 +5,7 @@ import java.util.Comparator;
|
|||||||
public class IntervalableComparatorByPosition implements Comparator<Intervalable> {
|
public class IntervalableComparatorByPosition implements Comparator<Intervalable> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int compare(Intervalable intervalable, Intervalable intervalable2) {
|
public int compare(final Intervalable intervalable, final Intervalable intervalable2) {
|
||||||
return intervalable.getStart() - intervalable2.getStart();
|
return intervalable.getStart() - intervalable2.getStart();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -5,11 +5,13 @@ import java.util.Comparator;
|
|||||||
public class IntervalableComparatorBySize implements Comparator<Intervalable> {
|
public class IntervalableComparatorBySize implements Comparator<Intervalable> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int compare(Intervalable intervalable, Intervalable intervalable2) {
|
public int compare(final Intervalable intervalable, final Intervalable intervalable2) {
|
||||||
int comparison = intervalable2.size() - intervalable.size();
|
int comparison = intervalable2.size() - intervalable.size();
|
||||||
|
|
||||||
if (comparison == 0) {
|
if (comparison == 0) {
|
||||||
comparison = intervalable.getStart() - intervalable2.getStart();
|
comparison = intervalable.getStart() - intervalable2.getStart();
|
||||||
}
|
}
|
||||||
|
|
||||||
return comparison;
|
return comparison;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -20,5 +20,4 @@ public class Emit extends Interval implements Intervalable {
|
|||||||
public String toString() {
|
public String toString() {
|
||||||
return super.toString() + "=" + this.keyword;
|
return super.toString() + "=" + this.keyword;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,9 +2,9 @@ package org.ahocorasick.trie;
|
|||||||
|
|
||||||
public class MatchToken extends Token {
|
public class MatchToken extends Token {
|
||||||
|
|
||||||
private Emit emit;
|
private final Emit emit;
|
||||||
|
|
||||||
public MatchToken(String fragment, Emit emit) {
|
public MatchToken(final String fragment, final Emit emit) {
|
||||||
super(fragment);
|
super(fragment);
|
||||||
this.emit = emit;
|
this.emit = emit;
|
||||||
}
|
}
|
||||||
@ -18,5 +18,4 @@ public class MatchToken extends Token {
|
|||||||
public Emit getEmit() {
|
public Emit getEmit() {
|
||||||
return this.emit;
|
return this.emit;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,7 +6,7 @@ import java.util.*;
|
|||||||
* <p>
|
* <p>
|
||||||
* A state has various important tasks it must attend to:
|
* A state has various important tasks it must attend to:
|
||||||
* </p>
|
* </p>
|
||||||
* <p>
|
*
|
||||||
* <ul>
|
* <ul>
|
||||||
* <li>success; when a character points to another state, it must return that state</li>
|
* <li>success; when a character points to another state, it must return that state</li>
|
||||||
* <li>failure; when a character has no matching state, the algorithm must be able to fall back on a
|
* <li>failure; when a character has no matching state, the algorithm must be able to fall back on a
|
||||||
@ -14,7 +14,7 @@ import java.util.*;
|
|||||||
* <li>emits; when this state is passed and keywords have been matched, the matches must be
|
* <li>emits; when this state is passed and keywords have been matched, the matches must be
|
||||||
* 'emitted' so that they can be used later on.</li>
|
* 'emitted' so that they can be used later on.</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
* <p>
|
*
|
||||||
* <p>
|
* <p>
|
||||||
* The root state is special in the sense that it has no failure state; it cannot fail. If it 'fails'
|
* The root state is special in the sense that it has no failure state; it cannot fail. If it 'fails'
|
||||||
* it will still parse the next character and start from the root node. This ensures that the algorithm
|
* it will still parse the next character and start from the root node. This ensures that the algorithm
|
||||||
@ -25,30 +25,22 @@ import java.util.*;
|
|||||||
*/
|
*/
|
||||||
public class State {
|
public class State {
|
||||||
|
|
||||||
/**
|
/** effective the size of the keyword */
|
||||||
* effective the size of the keyword
|
|
||||||
*/
|
|
||||||
private final int depth;
|
private final int depth;
|
||||||
|
|
||||||
/**
|
/** only used for the root state to refer to itself in case no matches have been found */
|
||||||
* only used for the root state to refer to itself in case no matches have been found
|
|
||||||
*/
|
|
||||||
private final State rootState;
|
private final State rootState;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* referred to in the white paper as the 'goto' structure. From a state it is possible to go
|
* referred to in the white paper as the 'goto' structure. From a state it is possible to go
|
||||||
* to other states, depending on the character passed.
|
* to other states, depending on the character passed.
|
||||||
*/
|
*/
|
||||||
private final Map<Character, State> success = new HashMap<>();
|
private final Map<Character,State> success = new HashMap<>();
|
||||||
|
|
||||||
/**
|
/** if no matching states are found, the failure state will be returned */
|
||||||
* if no matching states are found, the failure state will be returned
|
|
||||||
*/
|
|
||||||
private State failure;
|
private State failure;
|
||||||
|
|
||||||
/**
|
/** whenever this state is reached, it will emit the matches keywords for future reference */
|
||||||
* whenever this state is reached, it will emit the matches keywords for future reference
|
|
||||||
*/
|
|
||||||
private Set<String> emits;
|
private Set<String> emits;
|
||||||
|
|
||||||
public State() {
|
public State() {
|
||||||
@ -74,11 +66,11 @@ public class State {
|
|||||||
return nextState(character, false);
|
return nextState(character, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public State nextStateIgnoreRootState(Character character) {
|
public State nextStateIgnoreRootState(final Character character) {
|
||||||
return nextState(character, true);
|
return nextState(character, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
public State addState(String keyword) {
|
public State addState(final String keyword ) {
|
||||||
State state = this;
|
State state = this;
|
||||||
|
|
||||||
for (final Character character : keyword.toCharArray()) {
|
for (final Character character : keyword.toCharArray()) {
|
||||||
@ -88,10 +80,10 @@ public class State {
|
|||||||
return state;
|
return state;
|
||||||
}
|
}
|
||||||
|
|
||||||
public State addState(Character character) {
|
public State addState(final Character character) {
|
||||||
State nextState = nextStateIgnoreRootState(character);
|
State nextState = nextStateIgnoreRootState(character);
|
||||||
if (nextState == null) {
|
if (nextState == null) {
|
||||||
nextState = new State(this.depth + 1);
|
nextState = new State(this.depth+1);
|
||||||
this.success.put(character, nextState);
|
this.success.put(character, nextState);
|
||||||
}
|
}
|
||||||
return nextState;
|
return nextState;
|
||||||
@ -101,21 +93,21 @@ public class State {
|
|||||||
return this.depth;
|
return this.depth;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void addEmit(String keyword) {
|
public void addEmit(final String keyword) {
|
||||||
if (this.emits == null) {
|
if (this.emits == null) {
|
||||||
this.emits = new TreeSet<>();
|
this.emits = new TreeSet<>();
|
||||||
}
|
}
|
||||||
this.emits.add(keyword);
|
this.emits.add(keyword);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void addEmit(Collection<String> emits) {
|
public void addEmit(final Collection<String> emits) {
|
||||||
for (String emit : emits) {
|
for (String emit : emits) {
|
||||||
addEmit(emit);
|
addEmit(emit);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public Collection<String> emit() {
|
public Collection<String> emit() {
|
||||||
return this.emits == null ? Collections.<String>emptyList() : this.emits;
|
return this.emits == null ? Collections.<String> emptyList() : this.emits;
|
||||||
}
|
}
|
||||||
|
|
||||||
public State failure() {
|
public State failure() {
|
||||||
|
|||||||
@ -1,13 +1,13 @@
|
|||||||
package org.ahocorasick.trie;
|
package org.ahocorasick.trie;
|
||||||
|
|
||||||
|
import static java.lang.Character.isAlphabetic;
|
||||||
import static java.lang.Character.isWhitespace;
|
import static java.lang.Character.isWhitespace;
|
||||||
|
import static java.lang.Character.toLowerCase;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Queue;
|
import java.util.Queue;
|
||||||
import java.util.concurrent.LinkedBlockingDeque;
|
import java.util.concurrent.LinkedBlockingDeque;
|
||||||
|
|
||||||
import org.ahocorasick.interval.IntervalTree;
|
import org.ahocorasick.interval.IntervalTree;
|
||||||
import org.ahocorasick.interval.Intervalable;
|
import org.ahocorasick.interval.Intervalable;
|
||||||
import org.ahocorasick.trie.handler.DefaultEmitHandler;
|
import org.ahocorasick.trie.handler.DefaultEmitHandler;
|
||||||
@ -34,44 +34,21 @@ public class Trie {
|
|||||||
* Used by the builder to add a text search keyword.
|
* Used by the builder to add a text search keyword.
|
||||||
*
|
*
|
||||||
* @param keyword The search term to add to the list of search terms.
|
* @param keyword The search term to add to the list of search terms.
|
||||||
|
*
|
||||||
* @throws NullPointerException if the keyword is null.
|
* @throws NullPointerException if the keyword is null.
|
||||||
*/
|
*/
|
||||||
private void addKeyword(String keyword) {
|
private void addKeyword( String keyword ) {
|
||||||
if (keyword.isEmpty()) {
|
if( keyword.length() > 0 ) {
|
||||||
return;
|
if( isCaseInsensitive() ) {
|
||||||
}
|
|
||||||
|
|
||||||
if (isCaseInsensitive()) {
|
|
||||||
keyword = keyword.toLowerCase();
|
keyword = keyword.toLowerCase();
|
||||||
}
|
}
|
||||||
|
|
||||||
addState(keyword).addEmit(keyword);
|
addState( keyword ).addEmit( keyword );
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delegates to addKeyword.
|
|
||||||
*
|
|
||||||
* @param keywords List of search term to add to the list of search terms.
|
|
||||||
*/
|
|
||||||
private void addKeywords(final String[] keywords) {
|
|
||||||
for (final String keyword : keywords) {
|
|
||||||
addKeyword(keyword);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private State addState( final String keyword ) {
|
||||||
* Delegates to addKeyword.
|
return getRootState().addState( keyword );
|
||||||
*
|
|
||||||
* @param keywords List of search term to add to the list of search terms.
|
|
||||||
*/
|
|
||||||
private void addKeywords(final Collection<String> keywords) {
|
|
||||||
for (final String keyword : keywords) {
|
|
||||||
addKeyword(keyword);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private State addState(final String keyword) {
|
|
||||||
return getRootState().addState(keyword);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Collection<Token> tokenize(final String text) {
|
public Collection<Token> tokenize(final String text) {
|
||||||
@ -95,12 +72,15 @@ public class Trie {
|
|||||||
return tokens;
|
return tokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Token createFragment(final Emit emit, final String text, final int lastCollectedPosition) {
|
private Token createFragment(
|
||||||
return new FragmentToken(text.substring(lastCollectedPosition + 1, emit == null ? text.length() : emit.getStart()));
|
final Emit emit,
|
||||||
|
final String text,
|
||||||
|
final int lastCollectedPosition) {
|
||||||
|
return new FragmentToken(text.substring(lastCollectedPosition+1, emit == null ? text.length() : emit.getStart()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private Token createMatch(Emit emit, String text) {
|
private Token createMatch(final Emit emit, final String text) {
|
||||||
return new MatchToken(text.substring(emit.getStart(), emit.getEnd() + 1), emit);
|
return new MatchToken(text.substring(emit.getStart(), emit.getEnd()+1), emit);
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
@ -119,7 +99,7 @@ public class Trie {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!trieConfig.isAllowOverlaps()) {
|
if (!trieConfig.isAllowOverlaps()) {
|
||||||
IntervalTree intervalTree = new IntervalTree((List<Intervalable>) (List<?>) collectedEmits);
|
IntervalTree intervalTree = new IntervalTree((List<Intervalable>)(List<?>)collectedEmits);
|
||||||
intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
|
intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -138,7 +118,7 @@ public class Trie {
|
|||||||
|
|
||||||
// TODO: Maybe lowercase the entire string at once?
|
// TODO: Maybe lowercase the entire string at once?
|
||||||
if (trieConfig.isCaseInsensitive()) {
|
if (trieConfig.isCaseInsensitive()) {
|
||||||
character = Character.toLowerCase(character);
|
character = toLowerCase(character);
|
||||||
}
|
}
|
||||||
|
|
||||||
currentState = getState(currentState, character);
|
currentState = getState(currentState, character);
|
||||||
@ -164,14 +144,14 @@ public class Trie {
|
|||||||
|
|
||||||
// TODO: Lowercase the entire string at once?
|
// TODO: Lowercase the entire string at once?
|
||||||
if (trieConfig.isCaseInsensitive()) {
|
if (trieConfig.isCaseInsensitive()) {
|
||||||
character = Character.toLowerCase(character);
|
character = toLowerCase(character);
|
||||||
}
|
}
|
||||||
|
|
||||||
currentState = getState(currentState, character);
|
currentState = getState(currentState, character);
|
||||||
Collection<String> emitStrs = currentState.emit();
|
Collection<String> emitStrs = currentState.emit();
|
||||||
|
|
||||||
if (emitStrs != null && !emitStrs.isEmpty()) {
|
if (emitStrs != null && !emitStrs.isEmpty()) {
|
||||||
for (String emitStr : emitStrs) {
|
for (final String emitStr : emitStrs) {
|
||||||
final Emit emit = new Emit(position - emitStr.length() + 1, position, emitStr);
|
final Emit emit = new Emit(position - emitStr.length() + 1, position, emitStr);
|
||||||
if (trieConfig.isOnlyWholeWords()) {
|
if (trieConfig.isOnlyWholeWords()) {
|
||||||
if (!isPartialMatch(text, emit)) {
|
if (!isPartialMatch(text, emit)) {
|
||||||
@ -190,9 +170,9 @@ public class Trie {
|
|||||||
|
|
||||||
private boolean isPartialMatch(final CharSequence searchText, final Emit emit) {
|
private boolean isPartialMatch(final CharSequence searchText, final Emit emit) {
|
||||||
return (emit.getStart() != 0 &&
|
return (emit.getStart() != 0 &&
|
||||||
Character.isAlphabetic(searchText.charAt(emit.getStart() - 1))) ||
|
isAlphabetic(searchText.charAt(emit.getStart() - 1))) ||
|
||||||
(emit.getEnd() + 1 != searchText.length() &&
|
(emit.getEnd() + 1 != searchText.length() &&
|
||||||
Character.isAlphabetic(searchText.charAt(emit.getEnd() + 1)));
|
isAlphabetic(searchText.charAt(emit.getEnd() + 1)));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void removePartialMatches(final CharSequence searchText, final List<Emit> collectedEmits) {
|
private void removePartialMatches(final CharSequence searchText, final List<Emit> collectedEmits) {
|
||||||
@ -226,15 +206,16 @@ public class Trie {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private State getState(State currentState, final Character character) {
|
private State getState(final State initialState, final Character character) {
|
||||||
State newCurrentState = currentState.nextState(character);
|
State currentState = initialState;
|
||||||
|
State updatedState = currentState.nextState(character);
|
||||||
|
|
||||||
while (newCurrentState == null) {
|
while (updatedState == null) {
|
||||||
currentState = currentState.failure();
|
currentState = currentState.failure();
|
||||||
newCurrentState = currentState.nextState(character);
|
updatedState = currentState.nextState(character);
|
||||||
}
|
}
|
||||||
|
|
||||||
return newCurrentState;
|
return updatedState;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void constructFailureStates() {
|
private void constructFailureStates() {
|
||||||
@ -267,7 +248,10 @@ public class Trie {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean storeEmits(final int position, final State currentState, final EmitHandler emitHandler) {
|
private boolean storeEmits(
|
||||||
|
final int position,
|
||||||
|
final State currentState,
|
||||||
|
final EmitHandler emitHandler) {
|
||||||
boolean emitted = false;
|
boolean emitted = false;
|
||||||
final Collection<String> emits = currentState.emit();
|
final Collection<String> emits = currentState.emit();
|
||||||
|
|
||||||
@ -291,7 +275,8 @@ public class Trie {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Provides a fluent interface for constructing Trie instances.
|
* Constructs a TrieBuilder instance for configuring the Trie using a fluent
|
||||||
|
* interface.
|
||||||
*
|
*
|
||||||
* @return The builder used to configure its Trie.
|
* @return The builder used to configure its Trie.
|
||||||
*/
|
*/
|
||||||
@ -299,6 +284,9 @@ public class Trie {
|
|||||||
return new TrieBuilder();
|
return new TrieBuilder();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides a fluent interface for constructing Trie instances.
|
||||||
|
*/
|
||||||
public static class TrieBuilder {
|
public static class TrieBuilder {
|
||||||
|
|
||||||
private final TrieConfig trieConfig = new TrieConfig();
|
private final TrieConfig trieConfig = new TrieConfig();
|
||||||
@ -308,18 +296,18 @@ public class Trie {
|
|||||||
/**
|
/**
|
||||||
* Default (empty) constructor.
|
* Default (empty) constructor.
|
||||||
*/
|
*/
|
||||||
private TrieBuilder() {
|
private TrieBuilder() {}
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Adds a keyword to the Trie's list of text search keywords.
|
* Adds a keyword to the Trie's list of text search keywords.
|
||||||
*
|
*
|
||||||
* @param keyword The keyword to add to the list.
|
* @param keyword The keyword to add to the list.
|
||||||
|
*
|
||||||
* @return This builder.
|
* @return This builder.
|
||||||
* @throws NullPointerException if the keyword is null.
|
* @throws NullPointerException if the keyword is null.
|
||||||
*/
|
*/
|
||||||
public TrieBuilder addKeyword(final String keyword) {
|
public TrieBuilder addKeyword(final CharSequence keyword) {
|
||||||
this.trie.addKeyword(keyword);
|
getTrie().addKeyword( keyword.toString() );
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -327,10 +315,14 @@ public class Trie {
|
|||||||
* Adds a list of keywords to the Trie's list of text search keywords.
|
* Adds a list of keywords to the Trie's list of text search keywords.
|
||||||
*
|
*
|
||||||
* @param keywords The keywords to add to the list.
|
* @param keywords The keywords to add to the list.
|
||||||
|
*
|
||||||
* @return This builder.
|
* @return This builder.
|
||||||
*/
|
*/
|
||||||
public TrieBuilder addKeywords(final String... keywords) {
|
public TrieBuilder addKeywords(final CharSequence... keywords) {
|
||||||
this.trie.addKeywords(keywords);
|
for( final CharSequence keyword : keywords ) {
|
||||||
|
addKeyword( keyword );
|
||||||
|
}
|
||||||
|
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -338,21 +330,21 @@ public class Trie {
|
|||||||
* Adds a list of keywords to the Trie's list of text search keywords.
|
* Adds a list of keywords to the Trie's list of text search keywords.
|
||||||
*
|
*
|
||||||
* @param keywords The keywords to add to the list.
|
* @param keywords The keywords to add to the list.
|
||||||
|
*
|
||||||
* @return This builder.
|
* @return This builder.
|
||||||
*/
|
*/
|
||||||
public TrieBuilder addKeywords(final Collection<String> keywords) {
|
public TrieBuilder addKeywords(final Collection<CharSequence> keywords) {
|
||||||
this.trie.addKeywords(keywords);
|
return addKeywords( keywords.toArray( new CharSequence[ keywords.size() ] ) );
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Configure the Trie to ignore case when searching for keywords in
|
* Configure the Trie to ignore case when searching for keywords in the
|
||||||
* the text.
|
* text.
|
||||||
*
|
*
|
||||||
* @return This builder.
|
* @return This builder.
|
||||||
*/
|
*/
|
||||||
public TrieBuilder ignoreCase() {
|
public TrieBuilder ignoreCase() {
|
||||||
this.trieConfig.setCaseInsensitive(true);
|
getTrieConfig().setCaseInsensitive(true);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -362,7 +354,7 @@ public class Trie {
|
|||||||
* @return This builder.
|
* @return This builder.
|
||||||
*/
|
*/
|
||||||
public TrieBuilder ignoreOverlaps() {
|
public TrieBuilder ignoreOverlaps() {
|
||||||
this.trieConfig.setAllowOverlaps(false);
|
getTrieConfig().setAllowOverlaps(false);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -372,7 +364,7 @@ public class Trie {
|
|||||||
* @return This builder.
|
* @return This builder.
|
||||||
*/
|
*/
|
||||||
public TrieBuilder onlyWholeWords() {
|
public TrieBuilder onlyWholeWords() {
|
||||||
this.trieConfig.setOnlyWholeWords(true);
|
getTrieConfig().setOnlyWholeWords(true);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -384,33 +376,47 @@ public class Trie {
|
|||||||
* @return This builder.
|
* @return This builder.
|
||||||
*/
|
*/
|
||||||
public TrieBuilder onlyWholeWordsWhiteSpaceSeparated() {
|
public TrieBuilder onlyWholeWordsWhiteSpaceSeparated() {
|
||||||
this.trieConfig.setOnlyWholeWordsWhiteSpaceSeparated(true);
|
getTrieConfig().setOnlyWholeWordsWhiteSpaceSeparated(true);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Configure the Trie to stop after the first keyword is found in the
|
* Configure the Trie to stop searching for matches after the first
|
||||||
* text.
|
* keyword is found in the text.
|
||||||
*
|
*
|
||||||
* @return This builder.
|
* @return This builder.
|
||||||
*/
|
*/
|
||||||
public TrieBuilder stopOnHit() {
|
public TrieBuilder onlyFirstMatch() {
|
||||||
trie.trieConfig.setStopOnHit(true);
|
getTrieConfig().setStopOnHit(true);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Configure the Trie based on the builder settings.
|
* Construct the Trie using the builder settings.
|
||||||
*
|
*
|
||||||
* @return The configured Trie.
|
* @return The configured Trie.
|
||||||
*/
|
*/
|
||||||
public Trie build() {
|
public Trie build() {
|
||||||
this.trie.constructFailureStates();
|
getTrie().constructFailureStates();
|
||||||
|
return getTrie();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Trie getTrie() {
|
||||||
return this.trie;
|
return this.trie;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private TrieConfig getTrieConfig() {
|
||||||
|
return this.trieConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @deprecated Use onlyFirstMatch()
|
||||||
|
*/
|
||||||
|
public TrieBuilder stopOnHit() {
|
||||||
|
return onlyFirstMatch();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return This builder.
|
|
||||||
* @deprecated Use ignoreCase()
|
* @deprecated Use ignoreCase()
|
||||||
*/
|
*/
|
||||||
public TrieBuilder caseInsensitive() {
|
public TrieBuilder caseInsensitive() {
|
||||||
@ -418,7 +424,6 @@ public class Trie {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return This builder.
|
|
||||||
* @deprecated Use ignoreOverlaps()
|
* @deprecated Use ignoreOverlaps()
|
||||||
*/
|
*/
|
||||||
public TrieBuilder removeOverlaps() {
|
public TrieBuilder removeOverlaps() {
|
||||||
|
|||||||
@ -1,21 +1,19 @@
|
|||||||
package org.ahocorasick.trie.handler;
|
package org.ahocorasick.trie.handler;
|
||||||
|
|
||||||
import org.ahocorasick.trie.Emit;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import org.ahocorasick.trie.Emit;
|
||||||
|
|
||||||
public class DefaultEmitHandler implements EmitHandler {
|
public class DefaultEmitHandler implements EmitHandler {
|
||||||
|
|
||||||
private List<Emit> emits = new ArrayList<>();
|
private final List<Emit> emits = new ArrayList<>();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void emit(Emit emit) {
|
public void emit(final Emit emit) {
|
||||||
this.emits.add(emit);
|
this.emits.add(emit);
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Emit> getEmits() {
|
public List<Emit> getEmits() {
|
||||||
return this.emits;
|
return this.emits;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -44,7 +44,7 @@ public class IntervalTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void comparable() {
|
public void comparable() {
|
||||||
Set<Interval> intervals = new TreeSet<Interval>();
|
Set<Interval> intervals = new TreeSet<>();
|
||||||
intervals.add(new Interval(4, 6));
|
intervals.add(new Interval(4, 6));
|
||||||
intervals.add(new Interval(2, 7));
|
intervals.add(new Interval(2, 7));
|
||||||
intervals.add(new Interval(3, 4));
|
intervals.add(new Interval(3, 4));
|
||||||
|
|||||||
@ -12,7 +12,7 @@ public class IntervalTreeTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void findOverlaps() {
|
public void findOverlaps() {
|
||||||
List<Intervalable> intervals = new ArrayList<Intervalable>();
|
List<Intervalable> intervals = new ArrayList<>();
|
||||||
intervals.add(new Interval(0, 2));
|
intervals.add(new Interval(0, 2));
|
||||||
intervals.add(new Interval(1, 3));
|
intervals.add(new Interval(1, 3));
|
||||||
intervals.add(new Interval(2, 4));
|
intervals.add(new Interval(2, 4));
|
||||||
@ -30,7 +30,7 @@ public class IntervalTreeTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void removeOverlaps() {
|
public void removeOverlaps() {
|
||||||
List<Intervalable> intervals = new ArrayList<Intervalable>();
|
List<Intervalable> intervals = new ArrayList<>();
|
||||||
intervals.add(new Interval(0, 2));
|
intervals.add(new Interval(0, 2));
|
||||||
intervals.add(new Interval(4, 5));
|
intervals.add(new Interval(4, 5));
|
||||||
intervals.add(new Interval(2, 10));
|
intervals.add(new Interval(2, 10));
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import org.junit.Test;
|
|||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import static java.util.Collections.sort;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static junit.framework.Assert.assertEquals;
|
import static junit.framework.Assert.assertEquals;
|
||||||
@ -12,11 +13,11 @@ public class IntervalableComparatorByPositionTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void sortOnPosition() {
|
public void sortOnPosition() {
|
||||||
List<Intervalable> intervals = new ArrayList<Intervalable>();
|
List<Intervalable> intervals = new ArrayList<>();
|
||||||
intervals.add(new Interval(4, 5));
|
intervals.add(new Interval(4,5));
|
||||||
intervals.add(new Interval(1, 4));
|
intervals.add(new Interval(1,4));
|
||||||
intervals.add(new Interval(3, 8));
|
intervals.add(new Interval(3,8));
|
||||||
Collections.sort(intervals, new IntervalableComparatorByPosition());
|
sort(intervals, new IntervalableComparatorByPosition());
|
||||||
assertEquals(4, intervals.get(0).size());
|
assertEquals(4, intervals.get(0).size());
|
||||||
assertEquals(6, intervals.get(1).size());
|
assertEquals(6, intervals.get(1).size());
|
||||||
assertEquals(2, intervals.get(2).size());
|
assertEquals(2, intervals.get(2).size());
|
||||||
|
|||||||
@ -4,6 +4,8 @@ import org.junit.Test;
|
|||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import static java.util.Collections.sort;
|
||||||
|
import static java.util.Collections.sort;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static junit.framework.Assert.assertEquals;
|
import static junit.framework.Assert.assertEquals;
|
||||||
@ -12,11 +14,11 @@ public class IntervalableComparatorBySizeTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void sortOnSize() {
|
public void sortOnSize() {
|
||||||
List<Intervalable> intervals = new ArrayList<Intervalable>();
|
List<Intervalable> intervals = new ArrayList<>();
|
||||||
intervals.add(new Interval(4, 5));
|
intervals.add(new Interval(4,5));
|
||||||
intervals.add(new Interval(1, 4));
|
intervals.add(new Interval(1,4));
|
||||||
intervals.add(new Interval(3, 8));
|
intervals.add(new Interval(3,8));
|
||||||
Collections.sort(intervals, new IntervalableComparatorBySize());
|
sort(intervals, new IntervalableComparatorBySize());
|
||||||
assertEquals(6, intervals.get(0).size());
|
assertEquals(6, intervals.get(0).size());
|
||||||
assertEquals(4, intervals.get(1).size());
|
assertEquals(4, intervals.get(1).size());
|
||||||
assertEquals(2, intervals.get(2).size());
|
assertEquals(2, intervals.get(2).size());
|
||||||
@ -24,10 +26,10 @@ public class IntervalableComparatorBySizeTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void sortOnSizeThenPosition() {
|
public void sortOnSizeThenPosition() {
|
||||||
List<Intervalable> intervals = new ArrayList<Intervalable>();
|
List<Intervalable> intervals = new ArrayList<>();
|
||||||
intervals.add(new Interval(4, 7));
|
intervals.add(new Interval(4,7));
|
||||||
intervals.add(new Interval(2, 5));
|
intervals.add(new Interval(2,5));
|
||||||
Collections.sort(intervals, new IntervalableComparatorBySize());
|
sort(intervals, new IntervalableComparatorBySize());
|
||||||
assertEquals(2, intervals.get(0).getStart());
|
assertEquals(2, intervals.get(0).getStart());
|
||||||
assertEquals(4, intervals.get(1).getStart());
|
assertEquals(4, intervals.get(1).getStart());
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,13 +5,11 @@ import java.util.Collection;
|
|||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.ThreadLocalRandom;
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
|
import static java.util.concurrent.ThreadLocalRandom.current;
|
||||||
import static junit.framework.Assert.assertEquals;
|
import static junit.framework.Assert.assertEquals;
|
||||||
|
import static org.ahocorasick.trie.Trie.builder;
|
||||||
import org.ahocorasick.trie.handler.EmitHandler;
|
import org.ahocorasick.trie.handler.EmitHandler;
|
||||||
|
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
public class TrieTest {
|
public class TrieTest {
|
||||||
@ -37,7 +35,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void keywordAndTextAreTheSame() {
|
public void keywordAndTextAreTheSame() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeyword(ALPHABET[0])
|
.addKeyword(ALPHABET[0])
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText(ALPHABET[0]);
|
Collection<Emit> emits = trie.parseText(ALPHABET[0]);
|
||||||
@ -47,7 +45,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void keywordAndTextAreTheSameFirstMatch() {
|
public void keywordAndTextAreTheSameFirstMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeyword(ALPHABET[0])
|
.addKeyword(ALPHABET[0])
|
||||||
.build();
|
.build();
|
||||||
Emit firstMatch = trie.firstMatch(ALPHABET[0]);
|
Emit firstMatch = trie.firstMatch(ALPHABET[0]);
|
||||||
@ -56,7 +54,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void textIsLongerThanKeyword() {
|
public void textIsLongerThanKeyword() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeyword(ALPHABET[0])
|
.addKeyword(ALPHABET[0])
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText(" " + ALPHABET[0]);
|
Collection<Emit> emits = trie.parseText(" " + ALPHABET[0]);
|
||||||
@ -66,7 +64,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void textIsLongerThanKeywordFirstMatch() {
|
public void textIsLongerThanKeywordFirstMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeyword(ALPHABET[0])
|
.addKeyword(ALPHABET[0])
|
||||||
.build();
|
.build();
|
||||||
Emit firstMatch = trie.firstMatch(" " + ALPHABET[0]);
|
Emit firstMatch = trie.firstMatch(" " + ALPHABET[0]);
|
||||||
@ -75,7 +73,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void variousKeywordsOneMatch() {
|
public void variousKeywordsOneMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(ALPHABET)
|
.addKeywords(ALPHABET)
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText("bcd");
|
Collection<Emit> emits = trie.parseText("bcd");
|
||||||
@ -85,7 +83,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void variousKeywordsFirstMatch() {
|
public void variousKeywordsFirstMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(ALPHABET)
|
.addKeywords(ALPHABET)
|
||||||
.build();
|
.build();
|
||||||
Emit firstMatch = trie.firstMatch("bcd");
|
Emit firstMatch = trie.firstMatch("bcd");
|
||||||
@ -94,7 +92,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void ushersTestAndStopOnHit() {
|
public void ushersTestAndStopOnHit() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(PRONOUNS)
|
.addKeywords(PRONOUNS)
|
||||||
.stopOnHit()
|
.stopOnHit()
|
||||||
.build();
|
.build();
|
||||||
@ -107,7 +105,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void ushersTest() {
|
public void ushersTest() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(PRONOUNS)
|
.addKeywords(PRONOUNS)
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText("ushers");
|
Collection<Emit> emits = trie.parseText("ushers");
|
||||||
@ -120,7 +118,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void ushersTestWithCapitalKeywords() {
|
public void ushersTestWithCapitalKeywords() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.ignoreCase()
|
.ignoreCase()
|
||||||
.addKeyword("HERS")
|
.addKeyword("HERS")
|
||||||
.addKeyword("HIS")
|
.addKeyword("HIS")
|
||||||
@ -137,7 +135,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void ushersTestFirstMatch() {
|
public void ushersTestFirstMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(PRONOUNS)
|
.addKeywords(PRONOUNS)
|
||||||
.build();
|
.build();
|
||||||
Emit firstMatch = trie.firstMatch("ushers");
|
Emit firstMatch = trie.firstMatch("ushers");
|
||||||
@ -146,7 +144,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void ushersTestByCallback() {
|
public void ushersTestByCallback() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(PRONOUNS)
|
.addKeywords(PRONOUNS)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
@ -168,7 +166,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void misleadingTest() {
|
public void misleadingTest() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeyword("hers")
|
.addKeyword("hers")
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText("h he her hers");
|
Collection<Emit> emits = trie.parseText("h he her hers");
|
||||||
@ -178,7 +176,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void misleadingTestFirstMatch() {
|
public void misleadingTestFirstMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeyword("hers")
|
.addKeyword("hers")
|
||||||
.build();
|
.build();
|
||||||
Emit firstMatch = trie.firstMatch("h he her hers");
|
Emit firstMatch = trie.firstMatch("h he her hers");
|
||||||
@ -187,7 +185,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void recipes() {
|
public void recipes() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(FOOD)
|
.addKeywords(FOOD)
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText("2 cauliflowers, 3 tomatoes, 4 slices of veal, 100g broccoli");
|
Collection<Emit> emits = trie.parseText("2 cauliflowers, 3 tomatoes, 4 slices of veal, 100g broccoli");
|
||||||
@ -200,7 +198,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void recipesFirstMatch() {
|
public void recipesFirstMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(FOOD)
|
.addKeywords(FOOD)
|
||||||
.build();
|
.build();
|
||||||
Emit firstMatch = trie.firstMatch("2 cauliflowers, 3 tomatoes, 4 slices of veal, 100g broccoli");
|
Emit firstMatch = trie.firstMatch("2 cauliflowers, 3 tomatoes, 4 slices of veal, 100g broccoli");
|
||||||
@ -210,7 +208,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void longAndShortOverlappingMatch() {
|
public void longAndShortOverlappingMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeyword("he")
|
.addKeyword("he")
|
||||||
.addKeyword("hehehehe")
|
.addKeyword("hehehehe")
|
||||||
.build();
|
.build();
|
||||||
@ -227,7 +225,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void nonOverlapping() {
|
public void nonOverlapping() {
|
||||||
Trie trie = Trie.builder().removeOverlaps()
|
Trie trie = builder().removeOverlaps()
|
||||||
.addKeyword("ab")
|
.addKeyword("ab")
|
||||||
.addKeyword("cba")
|
.addKeyword("cba")
|
||||||
.addKeyword("ababc")
|
.addKeyword("ababc")
|
||||||
@ -242,7 +240,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void nonOverlappingFirstMatch() {
|
public void nonOverlappingFirstMatch() {
|
||||||
Trie trie = Trie.builder().removeOverlaps()
|
Trie trie = builder().removeOverlaps()
|
||||||
.addKeyword("ab")
|
.addKeyword("ab")
|
||||||
.addKeyword("cba")
|
.addKeyword("cba")
|
||||||
.addKeyword("ababc")
|
.addKeyword("ababc")
|
||||||
@ -254,7 +252,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void containsMatch() {
|
public void containsMatch() {
|
||||||
Trie trie = Trie.builder().removeOverlaps()
|
Trie trie = builder().removeOverlaps()
|
||||||
.addKeyword("ab")
|
.addKeyword("ab")
|
||||||
.addKeyword("cba")
|
.addKeyword("cba")
|
||||||
.addKeyword("ababc")
|
.addKeyword("ababc")
|
||||||
@ -264,7 +262,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void startOfChurchillSpeech() {
|
public void startOfChurchillSpeech() {
|
||||||
Trie trie = Trie.builder().removeOverlaps()
|
Trie trie = builder().removeOverlaps()
|
||||||
.addKeyword("T")
|
.addKeyword("T")
|
||||||
.addKeyword("u")
|
.addKeyword("u")
|
||||||
.addKeyword("ur")
|
.addKeyword("ur")
|
||||||
@ -282,7 +280,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void partialMatch() {
|
public void partialMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.onlyWholeWords()
|
.onlyWholeWords()
|
||||||
.addKeyword("sugar")
|
.addKeyword("sugar")
|
||||||
.build();
|
.build();
|
||||||
@ -293,7 +291,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void partialMatchFirstMatch() {
|
public void partialMatchFirstMatch() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.onlyWholeWords()
|
.onlyWholeWords()
|
||||||
.addKeyword("sugar")
|
.addKeyword("sugar")
|
||||||
.build();
|
.build();
|
||||||
@ -304,7 +302,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void tokenizeFullSentence() {
|
public void tokenizeFullSentence() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(GREEK_LETTERS)
|
.addKeywords(GREEK_LETTERS)
|
||||||
.build();
|
.build();
|
||||||
Collection<Token> tokens = trie.tokenize("Hear: Alpha team first, Beta from the rear, Gamma in reserve");
|
Collection<Token> tokens = trie.tokenize("Hear: Alpha team first, Beta from the rear, Gamma in reserve");
|
||||||
@ -322,7 +320,7 @@ public class TrieTest {
|
|||||||
// @see https://github.com/robert-bor/aho-corasick/issues/5
|
// @see https://github.com/robert-bor/aho-corasick/issues/5
|
||||||
@Test
|
@Test
|
||||||
public void testStringIndexOutOfBoundsException() {
|
public void testStringIndexOutOfBoundsException() {
|
||||||
Trie trie = Trie.builder().ignoreCase().onlyWholeWords()
|
Trie trie = builder().ignoreCase().onlyWholeWords()
|
||||||
.addKeywords(UNICODE)
|
.addKeywords(UNICODE)
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
|
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
|
||||||
@ -336,7 +334,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testIgnoreCase() {
|
public void testIgnoreCase() {
|
||||||
Trie trie = Trie.builder().ignoreCase()
|
Trie trie = builder().ignoreCase()
|
||||||
.addKeywords(UNICODE)
|
.addKeywords(UNICODE)
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
|
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
|
||||||
@ -350,7 +348,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testIgnoreCaseFirstMatch() {
|
public void testIgnoreCaseFirstMatch() {
|
||||||
Trie trie = Trie.builder().ignoreCase()
|
Trie trie = builder().ignoreCase()
|
||||||
.addKeywords(UNICODE)
|
.addKeywords(UNICODE)
|
||||||
.build();
|
.build();
|
||||||
Emit firstMatch = trie.firstMatch("TurninG OnCe AgAiN BÖRKÜ");
|
Emit firstMatch = trie.firstMatch("TurninG OnCe AgAiN BÖRKÜ");
|
||||||
@ -360,7 +358,7 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void tokenizeTokensInSequence() {
|
public void tokenizeTokensInSequence() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.addKeywords(GREEK_LETTERS)
|
.addKeywords(GREEK_LETTERS)
|
||||||
.build();
|
.build();
|
||||||
Collection<Token> tokens = trie.tokenize("Alpha Beta Gamma");
|
Collection<Token> tokens = trie.tokenize("Alpha Beta Gamma");
|
||||||
@ -370,7 +368,7 @@ public class TrieTest {
|
|||||||
// @see https://github.com/robert-bor/aho-corasick/issues/7
|
// @see https://github.com/robert-bor/aho-corasick/issues/7
|
||||||
@Test
|
@Test
|
||||||
public void testZeroLength() {
|
public void testZeroLength() {
|
||||||
Trie trie = Trie.builder().ignoreOverlaps().onlyWholeWords().ignoreCase()
|
Trie trie = builder().ignoreOverlaps().onlyWholeWords().ignoreCase()
|
||||||
.addKeyword("")
|
.addKeyword("")
|
||||||
.build();
|
.build();
|
||||||
trie.tokenize("Try a natural lip and subtle bronzer to keep all the focus on those big bright eyes with NARS Eyeshadow Duo in Rated R And the winner is... Boots No7 Advanced Renewal Anti-ageing Glycolic Peel Kit ($25 amazon.com) won most-appealing peel.");
|
trie.tokenize("Try a natural lip and subtle bronzer to keep all the focus on those big bright eyes with NARS Eyeshadow Duo in Rated R And the winner is... Boots No7 Advanced Renewal Anti-ageing Glycolic Peel Kit ($25 amazon.com) won most-appealing peel.");
|
||||||
@ -381,7 +379,7 @@ public class TrieTest {
|
|||||||
public void testUnicode1() {
|
public void testUnicode1() {
|
||||||
String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char
|
String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char
|
||||||
assertEquals("THIS", target.substring(5, 9)); // Java does it the right way
|
assertEquals("THIS", target.substring(5, 9)); // Java does it the right way
|
||||||
Trie trie = Trie.builder().ignoreCase().onlyWholeWords()
|
Trie trie = builder().ignoreCase().onlyWholeWords()
|
||||||
.addKeyword("this")
|
.addKeyword("this")
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText(target);
|
Collection<Emit> emits = trie.parseText(target);
|
||||||
@ -394,7 +392,7 @@ public class TrieTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testUnicode2() {
|
public void testUnicode2() {
|
||||||
String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char
|
String target = "LİKE THIS"; // The second character ('İ') is Unicode, which was read by AC as a 2-byte char
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.ignoreCase()
|
.ignoreCase()
|
||||||
.onlyWholeWords()
|
.onlyWholeWords()
|
||||||
.addKeyword("this")
|
.addKeyword("this")
|
||||||
@ -406,11 +404,11 @@ public class TrieTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testPartialMatchWhiteSpaces() {
|
public void testPartialMatchWhiteSpaces() {
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.onlyWholeWordsWhiteSpaceSeparated()
|
.onlyWholeWordsWhiteSpaceSeparated()
|
||||||
.addKeyword("#sugar-123")
|
.addKeyword("#sugar-123")
|
||||||
.build();
|
.build();
|
||||||
Collection<Emit> emits = trie.parseText("#sugar-123 #sugar-1234"); // left, middle, right test
|
Collection < Emit > emits = trie.parseText("#sugar-123 #sugar-1234"); // left, middle, right test
|
||||||
assertEquals(1, emits.size()); // Match must not be made
|
assertEquals(1, emits.size()); // Match must not be made
|
||||||
checkEmit(emits.iterator().next(), 0, 9, "#sugar-123");
|
checkEmit(emits.iterator().next(), 0, 9, "#sugar-123");
|
||||||
}
|
}
|
||||||
@ -419,19 +417,19 @@ public class TrieTest {
|
|||||||
public void testLargeString() {
|
public void testLargeString() {
|
||||||
final int interval = 100;
|
final int interval = 100;
|
||||||
final int textSize = 1000000;
|
final int textSize = 1000000;
|
||||||
final String keyword = FOOD[1];
|
final String keyword = FOOD[ 1 ];
|
||||||
final StringBuilder text = randomNumbers(textSize);
|
final StringBuilder text = randomNumbers( textSize );
|
||||||
|
|
||||||
injectKeyword(text, keyword, interval);
|
injectKeyword( text, keyword, interval );
|
||||||
|
|
||||||
Trie trie = Trie.builder()
|
Trie trie = builder()
|
||||||
.onlyWholeWords()
|
.onlyWholeWords()
|
||||||
.addKeyword(keyword)
|
.addKeyword( keyword )
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
final Collection<Emit> emits = trie.parseText(text);
|
final Collection<Emit> emits = trie.parseText( text );
|
||||||
|
|
||||||
assertEquals(textSize / interval, emits.size());
|
assertEquals( textSize / interval, emits.size() );
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -440,11 +438,11 @@ public class TrieTest {
|
|||||||
* @param count The number of numbers to generate.
|
* @param count The number of numbers to generate.
|
||||||
* @return A character sequence filled with random digits.
|
* @return A character sequence filled with random digits.
|
||||||
*/
|
*/
|
||||||
private StringBuilder randomNumbers(int count) {
|
private StringBuilder randomNumbers( final int count ) {
|
||||||
final StringBuilder sb = new StringBuilder(count);
|
final StringBuilder sb = new StringBuilder( count );
|
||||||
|
|
||||||
while (--count > 0) {
|
for( int i = count - 1; i >= 0; i-- ) {
|
||||||
sb.append(randomInt(0, 10));
|
sb.append( randomInt( 0, 10 ) );
|
||||||
}
|
}
|
||||||
|
|
||||||
return sb;
|
return sb;
|
||||||
@ -461,15 +459,15 @@ public class TrieTest {
|
|||||||
private void injectKeyword(
|
private void injectKeyword(
|
||||||
final StringBuilder source,
|
final StringBuilder source,
|
||||||
final String keyword,
|
final String keyword,
|
||||||
final int interval) {
|
final int interval ) {
|
||||||
final int length = source.length();
|
final int length = source.length();
|
||||||
for (int i = 0; i < length; i += interval) {
|
for( int i = 0; i < length; i += interval ) {
|
||||||
source.replace(i, i + keyword.length(), keyword);
|
source.replace( i, i + keyword.length(), keyword );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private int randomInt(final int min, final int max) {
|
private int randomInt( final int min, final int max ) {
|
||||||
return ThreadLocalRandom.current().nextInt(min, max);
|
return current().nextInt( min, max );
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkEmit(Emit next, int expectedStart, int expectedEnd, String expectedKeyword) {
|
private void checkEmit(Emit next, int expectedStart, int expectedEnd, String expectedKeyword) {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user